lkness/bi
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

优化

Browse Source
This commit is contained in:
1340691923@qq.com 2022-03-10 17:10:33 +08:00
parent cf77900bc4
commit 39ae332a5b
2 changed files with 166 additions and 164 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
cmd/sinker/main.go
View File

@ -129,7 +129,7 @@ func main() {
markFn() markFn()
return return
} }
appid,err := strconv.Atoi(kafkaData.TableId) appid, err := strconv.Atoi(kafkaData.TableId)
if err != nil { if err != nil {
logs.Logger.Error("strconv.Atoi(kafkaData.TableId) Err", zap.Error(err)) logs.Logger.Error("strconv.Atoi(kafkaData.TableId) Err", zap.Error(err))
markFn() markFn()
@ -154,8 +154,8 @@ func main() {
panic(err) panic(err)
} }
parserPool,err := parser.NewParserPool("fastjson") parserPool, err := parser.NewParserPool("fastjson")
if err!=nil{ if err != nil {
panic(err) panic(err)
} }
@ -166,6 +166,7 @@ func main() {
func(msg model.InputMessage, markFn func()) { func(msg model.InputMessage, markFn func()) {
var kafkaData model.KafkaData var kafkaData model.KafkaData
err = json.Unmarshal(msg.Value, &kafkaData) err = json.Unmarshal(msg.Value, &kafkaData)
if err != nil { if err != nil {
logs.Logger.Error("json.Unmarshal Err", zap.Error(err)) logs.Logger.Error("json.Unmarshal Err", zap.Error(err))
@ -184,7 +185,7 @@ func main() {
tableId, _ := strconv.Atoi(kafkaData.TableId) tableId, _ := strconv.Atoi(kafkaData.TableId)
if kafkaData.EventName == ""{ if kafkaData.EventName == "" {
markFn() markFn()
return return
} }
@ -279,7 +280,6 @@ func main() {
return return
} }
//添加元数据 //添加元数据
if err := action.AddMetaEvent(kafkaData); err != nil { if err := action.AddMetaEvent(kafkaData); err != nil {
logs.Logger.Error("addMetaEvent err", zap.Error(err)) logs.Logger.Error("addMetaEvent err", zap.Error(err))

8
platform-basic-libs/service/consumer_data/reportdata2ck.go
View File

@ -48,8 +48,8 @@ func (this *ReportData2CK) Flush() (err error) {
rowsMap := map[string][][]interface{}{} rowsMap := map[string][][]interface{}{}
for bufferIndex := range this.buffer { for _,obj := range this.buffer {
for tableName := range this.buffer[bufferIndex] { for tableName,data := range obj {
rowArr := []interface{}{} rowArr := []interface{}{}
rows := [][]interface{}{} rows := [][]interface{}{}
if _, haveKey := rowsMap[tableName]; haveKey { if _, haveKey := rowsMap[tableName]; haveKey {
@ -59,7 +59,7 @@ func (this *ReportData2CK) Flush() (err error) {
} }
dims, _ := TableColumnMap.Load(tableName) dims, _ := TableColumnMap.Load(tableName)
for _, dim := range dims.([]*model2.ColumnWithType) { for _, dim := range dims.([]*model2.ColumnWithType) {
val := parser.GetValueByType(this.buffer[bufferIndex][tableName], dim) val := parser.GetValueByType(data, dim)
rowArr = append(rowArr, val) rowArr = append(rowArr, val)
} }
@ -68,6 +68,7 @@ func (this *ReportData2CK) Flush() (err error) {
} }
} }
bytesbuffer:=bytes.Buffer{} bytesbuffer:=bytes.Buffer{}
TableColumnMap.Range(func(tableName, value interface{}) bool { TableColumnMap.Range(func(tableName, value interface{}) bool {
@ -107,6 +108,7 @@ func (this *ReportData2CK) Flush() (err error) {
} }
defer stmt.Close() defer stmt.Close()
haveFail := false haveFail := false
for _, row := range rowsMap[tableName.(string)] { for _, row := range rowsMap[tableName.(string)] {
if _, err := stmt.Exec(row...); err != nil { if _, err := stmt.Exec(row...); err != nil {