Golang框架實(shí)戰(zhàn)-KisFlow流式計(jì)算框架專(zhuān)欄

Golang框架實(shí)戰(zhàn)-KisFlow流式計(jì)算框架(1)-概述
Golang框架實(shí)戰(zhàn)-KisFlow流式計(jì)算框架(2)-項(xiàng)目構(gòu)建/基礎(chǔ)模塊-(上)
Golang框架實(shí)戰(zhàn)-KisFlow流式計(jì)算框架(3)-項(xiàng)目構(gòu)建/基礎(chǔ)模塊-(下)
Golang框架實(shí)戰(zhàn)-KisFlow流式計(jì)算框架(4)-數(shù)據(jù)流
Golang框架實(shí)戰(zhàn)-KisFlow流式計(jì)算框架(5)-Function調(diào)度
Golang框架實(shí)戰(zhàn)-KisFlow流式計(jì)算框架(6)-Connector
Golang框架實(shí)戰(zhàn)-KisFlow流式計(jì)算框架(7)-配置導(dǎo)入與導(dǎo)出
Golang框架實(shí)戰(zhàn)-KisFlow流式計(jì)算框架(8)-KisFlow Action
Golang框架實(shí)戰(zhàn)-KisFlow流式計(jì)算框架(9)-Cache/Params 數(shù)據(jù)緩存與數(shù)據(jù)參數(shù)
Golang框架實(shí)戰(zhàn)-KisFlow流式計(jì)算框架(10)-Flow多副本
Golang框架實(shí)戰(zhàn)-KisFlow流式計(jì)算框架(11)-Prometheus Metrics統(tǒng)計(jì)
Golang框架實(shí)戰(zhàn)-KisFlow流式計(jì)算框架(12)-基于反射自適應(yīng)注冊(cè)FaaS形參類(lèi)型

案例：
KisFlow-Golang流式計(jì)算案例(一)快速開(kāi)始QuickStart
KisFlow-Golang流式計(jì)算案例(二)-Flow并流操作
KisFlow-Golang流式計(jì)算案例(二)-KisFlow在多協(xié)程中的應(yīng)用

DownLoad kis-flow source

$go get github.com/aceld/kis-flow

KisFlow with Kafka

案例源代碼
https://github.com/aceld/kis-flow-usage/tree/main/12-with_kafka

這里以github.com/segmentio/kafka-go 作為第三方Kafka Client SDK（開(kāi)發(fā)者也可以選擇其他kafka的go開(kāi)發(fā)工具）。

package main

import (
    "context"
    "fmt"
    "github.com/aceld/kis-flow/file"
    "github.com/aceld/kis-flow/kis"
    "github.com/segmentio/kafka-go"
    "sync"
    "time"
)

func main() {
    ctx := context.Background()

    // Load Configuration from file
    if err := file.ConfigImportYaml("conf/"); err != nil {
        panic(err)
    }

    // Get the flow
    flowOrg := kis.Pool().GetFlow("CalStuAvgScore")
    if flowOrg == nil {
        panic("flowOrg is nil")
    }

    // Create a new Kafka reader
    reader := kafka.NewReader(kafka.ReaderConfig{
        Brokers:     []string{"localhost:9092"},
        Topic:       "SourceStuScore",
        GroupID:     "group1",
        MinBytes:    10e3,                   // 10KB
        MaxBytes:    10e6,                   // 10MB
        MaxWait:     500 * time.Millisecond, // 最長(zhǎng)等待時(shí)間
        StartOffset: kafka.FirstOffset,
    })

    defer reader.Close()

    var wg sync.WaitGroup
    for i := 0; i < 5; i++ { // use 5 consumers to consume in parallel
        wg.Add(1)
        go func() {
            // fork a new flow for each consumer
            flowCopy := flowOrg.Fork(ctx)

            defer wg.Done()
            for {
                // Read a message from Kafka
                message, err := reader.ReadMessage(ctx)
                if err != nil {
                    fmt.Printf("error reading message: %v\n", err)
                    break
                }

                // Commit the message to the flow
                _ = flowCopy.CommitRow(string(message.Value))

                // Run the flow
                if err := flowCopy.Run(ctx); err != nil {
                    fmt.Println("err: ", err)
                    return
                }
            }
        }()
    }

    wg.Wait()

    return
}

func init() {
    // Register functions
    kis.Pool().FaaS("VerifyStu", VerifyStu)
    kis.Pool().FaaS("AvgStuScore", AvgStuScore)
    kis.Pool().FaaS("PrintStuAvgScore", PrintStuAvgScore)
}

KisFlow with Nsq

案例源代碼
https://github.com/aceld/kis-flow-usage/tree/main/13-with_nsq

本KisFlow消費(fèi)者以github.com/nsqio/go-nsq作為第三方SDK。

package main

import (
    "context"
    "fmt"
    "github.com/aceld/kis-flow/file"
    "github.com/aceld/kis-flow/kis"
    "github.com/nsqio/go-nsq"
)

func main() {
    ctx := context.Background()

    // Load Configuration from file
    if err := file.ConfigImportYaml("conf/"); err != nil {
        panic(err)
    }

    // Get the flow
    flowOrg := kis.Pool().GetFlow("CalStuAvgScore")
    if flowOrg == nil {
        panic("flowOrg is nil")
    }

    // Create a new NSQ consumer
    config := nsq.NewConfig()
    config.MaxInFlight = 5

    consumer, err := nsq.NewConsumer("SourceStuScore", "channel1", config)
    if err != nil {
        panic(err)
    }

    consumer.AddHandler(nsq.HandlerFunc(func(message *nsq.Message) error {
        // fork a new flow for each message
        flowCopy := flowOrg.Fork(ctx)

        // Commit the message to the flow
        _ = flowCopy.CommitRow(string(message.Body))

        // Run the flow
        if err := flowCopy.Run(ctx); err != nil {
            fmt.Println("err: ", err)
            return err
        }

        return nil
    }))

    err = consumer.ConnectToNSQLookupd("localhost:4161")
    if err != nil {
        panic(err)
    }

    defer consumer.Stop()

    select {}
}

func init() {
    // Register functions
    kis.Pool().FaaS("VerifyStu", VerifyStu)
    kis.Pool().FaaS("AvgStuScore", AvgStuScore)
    kis.Pool().FaaS("PrintStuAvgScore", PrintStuAvgScore)
}

KisFlow with RocketMQ

案例源代碼
https://github.com/aceld/kis-flow-usage/tree/main/14-with_rocketmq

以 github.com/apache/rocketmq-client-go 作為RocketMQ消費(fèi)者SDK。

package main

import (
    "context"
    "fmt"
    "github.com/aceld/kis-flow/file"
    "github.com/aceld/kis-flow/kis"
    "github.com/apache/rocketmq-client-go/v2"
    "github.com/apache/rocketmq-client-go/v2/consumer"
    "github.com/apache/rocketmq-client-go/v2/primitive"
)

func main() {
    // Load Configuration from file
    if err := file.ConfigImportYaml("conf/"); err != nil {
        panic(err)
    }

    // Get the flow
    myFloq := kis.Pool().GetFlow("CalStuAvgScore")
    if myFloq == nil {
        panic("myFloq is nil")
    }

    // Create a new RocketMQ consumer
    c, err := rocketmq.NewPushConsumer(
        consumer.WithGroupName("group1"),
        consumer.WithNameServer([]string{"localhost:9876"}),
    )
    if err != nil {
        panic(err)
    }

    err = c.Subscribe("SourceStuScore", consumer.MessageSelector{}, func(ctx context.Context, msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) {

        for _, msg := range msgs {
            // Commit the message to the flow
            _ = myFloq.CommitRow(string(msg.Body))

        }

        // Run the flow
        if err := myFloq.Run(ctx); err != nil {
            fmt.Println("err: ", err)
            return consumer.ConsumeRetryLater, err
        }

        return consumer.ConsumeSuccess, nil
    })
    if err != nil {
        panic(err)
    }

    err = c.Start()
    if err != nil {
        panic(err)
    }

    defer c.Shutdown()

    select {}
}

作者:劉丹冰Aceld github: https://github.com/aceld
KisFlow開(kāi)源項(xiàng)目地址：https://github.com/aceld/kis-flow

Golang框架實(shí)戰(zhàn)-KisFlow流式計(jì)算框架專(zhuān)欄

Golang框架實(shí)戰(zhàn)-KisFlow流式計(jì)算框架(1)-概述
Golang框架實(shí)戰(zhàn)-KisFlow流式計(jì)算框架(2)-項(xiàng)目構(gòu)建/基礎(chǔ)模塊-(上)
Golang框架實(shí)戰(zhàn)-KisFlow流式計(jì)算框架(3)-項(xiàng)目構(gòu)建/基礎(chǔ)模塊-(下)
Golang框架實(shí)戰(zhàn)-KisFlow流式計(jì)算框架(4)-數(shù)據(jù)流
Golang框架實(shí)戰(zhàn)-KisFlow流式計(jì)算框架(5)-Function調(diào)度
Golang框架實(shí)戰(zhàn)-KisFlow流式計(jì)算框架(6)-Connector
Golang框架實(shí)戰(zhàn)-KisFlow流式計(jì)算框架(7)-配置導(dǎo)入與導(dǎo)出
Golang框架實(shí)戰(zhàn)-KisFlow流式計(jì)算框架(8)-KisFlow Action
Golang框架實(shí)戰(zhàn)-KisFlow流式計(jì)算框架(9)-Cache/Params 數(shù)據(jù)緩存與數(shù)據(jù)參數(shù)
Golang框架實(shí)戰(zhàn)-KisFlow流式計(jì)算框架(10)-Flow多副本
Golang框架實(shí)戰(zhàn)-KisFlow流式計(jì)算框架(11)-Prometheus Metrics統(tǒng)計(jì)
Golang框架實(shí)戰(zhàn)-KisFlow流式計(jì)算框架(12)-基于反射自適應(yīng)注冊(cè)FaaS形參類(lèi)型

案例：
KisFlow-Golang流式計(jì)算案例(一)快速開(kāi)始QuickStart
KisFlow-Golang流式計(jì)算案例(二)-Flow并流操作
KisFlow-Golang流式計(jì)算案例(三)-KisFlow在多協(xié)程中的應(yīng)用