Golang框架實戰-KisFlow流式計算框架專欄
Golang框架實戰-KisFlow流式計算框架(1)-概述
Golang框架實戰-KisFlow流式計算框架(2)-項目構建/基礎模塊-(上)
Golang框架實戰-KisFlow流式計算框架(3)-項目構建/基礎模塊-(下)
Golang框架實戰-KisFlow流式計算框架(4)-數據流
Golang框架實戰-KisFlow流式計算框架(5)-Function調度
Golang框架實戰-KisFlow流式計算框架(6)-Connector
Golang框架實戰-KisFlow流式計算框架(7)-配置導入與導出
Golang框架實戰-KisFlow流式計算框架(8)-KisFlow Action
Golang框架實戰-KisFlow流式計算框架(9)-Cache/Params 數據緩存與數據參數
Golang框架實戰-KisFlow流式計算框架(10)-Flow多副本
Golang框架實戰-KisFlow流式計算框架(11)-Prometheus Metrics統計
Golang框架實戰-KisFlow流式計算框架(12)-基于反射自適應注冊FaaS形參類型
案例：
KisFlow-Golang流式計算案例(一)快速開始QuickStart
KisFlow-Golang流式計算案例(二)-Flow并流操作
KisFlow-Golang流式計算案例(二)-KisFlow在多協程中的應用
DownLoad kis-flow source
$go get github.com/aceld/kis-flow
KisFlow with Kafka
案例源代碼
https://github.com/aceld/kis-flow-usage/tree/main/12-with_kafka
這里以github.com/segmentio/kafka-go 作為第三方Kafka Client SDK（開發者也可以選擇其他kafka的go開發工具）。
package main
import (
"context"
"fmt"
"github.com/aceld/kis-flow/file"
"github.com/aceld/kis-flow/kis"
"github.com/segmentio/kafka-go"
"sync"
"time"
)
func main() {
ctx := context.Background()
// Load Configuration from file
if err := file.ConfigImportYaml("conf/"); err != nil {
panic(err)
}
// Get the flow
flowOrg := kis.Pool().GetFlow("CalStuAvgScore")
if flowOrg == nil {
panic("flowOrg is nil")
}
// Create a new Kafka reader
reader := kafka.NewReader(kafka.ReaderConfig{
Brokers: []string{"localhost:9092"},
Topic: "SourceStuScore",
GroupID: "group1",
MinBytes: 10e3, // 10KB
MaxBytes: 10e6, // 10MB
MaxWait: 500 * time.Millisecond, // 最長等待時間
StartOffset: kafka.FirstOffset,
})
defer reader.Close()
var wg sync.WaitGroup
for i := 0; i < 5; i++ { // use 5 consumers to consume in parallel
wg.Add(1)
go func() {
// fork a new flow for each consumer
flowCopy := flowOrg.Fork(ctx)
defer wg.Done()
for {
// Read a message from Kafka
message, err := reader.ReadMessage(ctx)
if err != nil {
fmt.Printf("error reading message: %v\n", err)
break
}
// Commit the message to the flow
_ = flowCopy.CommitRow(string(message.Value))
// Run the flow
if err := flowCopy.Run(ctx); err != nil {
fmt.Println("err: ", err)
return
}
}
}()
}
wg.Wait()
return
}
func init() {
// Register functions
kis.Pool().FaaS("VerifyStu", VerifyStu)
kis.Pool().FaaS("AvgStuScore", AvgStuScore)
kis.Pool().FaaS("PrintStuAvgScore", PrintStuAvgScore)
}
KisFlow with Nsq
案例源代碼
https://github.com/aceld/kis-flow-usage/tree/main/13-with_nsq
本KisFlow消費者以github.com/nsqio/go-nsq作為第三方SDK。
package main
import (
"context"
"fmt"
"github.com/aceld/kis-flow/file"
"github.com/aceld/kis-flow/kis"
"github.com/nsqio/go-nsq"
)
func main() {
ctx := context.Background()
// Load Configuration from file
if err := file.ConfigImportYaml("conf/"); err != nil {
panic(err)
}
// Get the flow
flowOrg := kis.Pool().GetFlow("CalStuAvgScore")
if flowOrg == nil {
panic("flowOrg is nil")
}
// Create a new NSQ consumer
config := nsq.NewConfig()
config.MaxInFlight = 5
consumer, err := nsq.NewConsumer("SourceStuScore", "channel1", config)
if err != nil {
panic(err)
}
consumer.AddHandler(nsq.HandlerFunc(func(message *nsq.Message) error {
// fork a new flow for each message
flowCopy := flowOrg.Fork(ctx)
// Commit the message to the flow
_ = flowCopy.CommitRow(string(message.Body))
// Run the flow
if err := flowCopy.Run(ctx); err != nil {
fmt.Println("err: ", err)
return err
}
return nil
}))
err = consumer.ConnectToNSQLookupd("localhost:4161")
if err != nil {
panic(err)
}
defer consumer.Stop()
select {}
}
func init() {
// Register functions
kis.Pool().FaaS("VerifyStu", VerifyStu)
kis.Pool().FaaS("AvgStuScore", AvgStuScore)
kis.Pool().FaaS("PrintStuAvgScore", PrintStuAvgScore)
}
KisFlow with RocketMQ
案例源代碼
https://github.com/aceld/kis-flow-usage/tree/main/14-with_rocketmq
以 github.com/apache/rocketmq-client-go 作為RocketMQ消費者SDK。
package main
import (
"context"
"fmt"
"github.com/aceld/kis-flow/file"
"github.com/aceld/kis-flow/kis"
"github.com/apache/rocketmq-client-go/v2"
"github.com/apache/rocketmq-client-go/v2/consumer"
"github.com/apache/rocketmq-client-go/v2/primitive"
)
func main() {
// Load Configuration from file
if err := file.ConfigImportYaml("conf/"); err != nil {
panic(err)
}
// Get the flow
myFloq := kis.Pool().GetFlow("CalStuAvgScore")
if myFloq == nil {
panic("myFloq is nil")
}
// Create a new RocketMQ consumer
c, err := rocketmq.NewPushConsumer(
consumer.WithGroupName("group1"),
consumer.WithNameServer([]string{"localhost:9876"}),
)
if err != nil {
panic(err)
}
err = c.Subscribe("SourceStuScore", consumer.MessageSelector{}, func(ctx context.Context, msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) {
for _, msg := range msgs {
// Commit the message to the flow
_ = myFloq.CommitRow(string(msg.Body))
}
// Run the flow
if err := myFloq.Run(ctx); err != nil {
fmt.Println("err: ", err)
return consumer.ConsumeRetryLater, err
}
return consumer.ConsumeSuccess, nil
})
if err != nil {
panic(err)
}
err = c.Start()
if err != nil {
panic(err)
}
defer c.Shutdown()
select {}
}
作者:劉丹冰Aceld github: https://github.com/aceld
KisFlow開源項目地址：https://github.com/aceld/kis-flow
Golang框架實戰-KisFlow流式計算框架專欄
Golang框架實戰-KisFlow流式計算框架(1)-概述
Golang框架實戰-KisFlow流式計算框架(2)-項目構建/基礎模塊-(上)
Golang框架實戰-KisFlow流式計算框架(3)-項目構建/基礎模塊-(下)
Golang框架實戰-KisFlow流式計算框架(4)-數據流
Golang框架實戰-KisFlow流式計算框架(5)-Function調度
Golang框架實戰-KisFlow流式計算框架(6)-Connector
Golang框架實戰-KisFlow流式計算框架(7)-配置導入與導出
Golang框架實戰-KisFlow流式計算框架(8)-KisFlow Action
Golang框架實戰-KisFlow流式計算框架(9)-Cache/Params 數據緩存與數據參數
Golang框架實戰-KisFlow流式計算框架(10)-Flow多副本
Golang框架實戰-KisFlow流式計算框架(11)-Prometheus Metrics統計
Golang框架實戰-KisFlow流式計算框架(12)-基于反射自適應注冊FaaS形參類型
案例：
KisFlow-Golang流式計算案例(一)快速開始QuickStart
KisFlow-Golang流式計算案例(二)-Flow并流操作
KisFlow-Golang流式計算案例(三)-KisFlow在多協程中的應用
