优化

cmd/sinker/action/action.go

@@ -99,7 +99,7 @@ func AddMetaEvent(kafkaData model.KafkaData) (err error) {
 
 func AddTableColumn(kafkaData model.KafkaData, failFunc func(data consumer_data.ReportAcceptStatusData), tableName string, ReqDataObject *parser.FastjsonMetric) (err error) {
 
-	dims, err := sinker.GetDims(model.GlobConfig.Comm.ClickHouse.DbName, tableName, nil, db.ClickHouseSqlx)
+	dims, err := sinker.GetDims(model.GlobConfig.Comm.ClickHouse.DbName, tableName, nil, db.ClickHouseSqlx,false)
 	if err != nil {
 		logs.Logger.Error("sinker.GetDims", zap.Error(err))
 		return

controller/report_controller.go

@@ -106,7 +106,7 @@ func (this ReportController) ReportAction(ctx *fasthttp.RequestCtx) {
 			return
 		}
 
-		dims, err := sinker.GetDims(model.GlobConfig.Comm.ClickHouse.DbName, kafkaData.GetTableName(), []string{}, db.ClickHouseSqlx)
+		dims, err := sinker.GetDims(model.GlobConfig.Comm.ClickHouse.DbName, kafkaData.GetTableName(), []string{}, db.ClickHouseSqlx,true)
 		if err != nil {
 			logs.Logger.Error("sinker.GetDims", zap.Error(err))
 			this.FastError(ctx, errors.New("服务异常"))

platform-basic-libs/service/consumer_data/real_time_warehousing.go

@@ -24,6 +24,7 @@ type RealTimeWarehousing struct {
 }
 
 func NewRealTimeWarehousing(batchSize, flushInterval int) *RealTimeWarehousing {
+	logs.Logger.Info("NewRealTimeWarehousing", zap.Int("batchSize", batchSize), zap.Int("flushInterval", flushInterval))
 	realTimeWarehousing := &RealTimeWarehousing{
 		buffer:        make([]*RealTimeWarehousingData, 0, batchSize),
 		bufferMutex:   new(sync.RWMutex),

platform-basic-libs/service/consumer_data/report_accpet_status.go

@@ -31,6 +31,7 @@ const FailStatus = 0
 const SuccessStatus = 1
 
 func NewReportAcceptStatus(batchSize int, flushInterval int) *ReportAcceptStatus {
+	logs.Logger.Info("NewReportAcceptStatus", zap.Int("batchSize", batchSize), zap.Int("flushInterval", flushInterval))
 	reportAcceptStatus := &ReportAcceptStatus{
 		buffer:        make([]*ReportAcceptStatusData, 0, batchSize),
 		bufferMutex:   new(sync.RWMutex),
@@ -85,10 +86,9 @@ func (this *ReportAcceptStatus) Flush() (err error) {
 	if err := tx.Commit(); err != nil {
 		logs.Logger.Error("入库数据状态出现错误", zap.Error(err))
 	} else {
-		lostTime := time.Now().Sub(startNow).String()
 		len := len(this.buffer)
 		if len > 0 {
-			logs.Logger.Info("入库数据状态成功", zap.String("所花时间", lostTime), zap.Int("数据长度为", len))
+			logs.Logger.Info("入库数据状态成功", zap.String("所花时间",  time.Now().Sub(startNow).String()), zap.Int("数据长度为", len))
 		}
 	}
 	stmt.Close()

platform-basic-libs/service/consumer_data/reportdata2ck.go

@@ -46,18 +46,19 @@ func (this *ReportData2CK) Flush() (err error) {
 	startNow := time.Now()
 
 	rowsMap := map[string][][]interface{}{}
+	rowArr := []interface{}{}
+	rows := [][]interface{}{}
 	for bufferIndex := range this.buffer {
 		for tableName := range this.buffer[bufferIndex] {
-			rows := [][]interface{}{}
+			rows := rows[0:0]
 			if _, haveKey := rowsMap[tableName]; haveKey {
 				rows = rowsMap[tableName]
 			} else {
 				rowsMap[tableName] = rows
 			}
-			v, _ := TableColumnMap.Load(tableName)
+			dims, _ := TableColumnMap.Load(tableName)
-			dims := v.([]*model2.ColumnWithType)
+			rowArr = rowArr[0:0]
-			var rowArr []interface{}
+			for _, dim := range dims.([]*model2.ColumnWithType) {
-			for _, dim := range dims {
 				val := parser.GetValueByType(this.buffer[bufferIndex][tableName], dim)
 				rowArr = append(rowArr, val)
 			}
@@ -68,11 +69,9 @@ func (this *ReportData2CK) Flush() (err error) {
 
 	bytesbuffer:=bytes.Buffer{}
 
-	TableColumnMap.Range(func(key, value interface{}) bool {
+	TableColumnMap.Range(func(tableName, value interface{}) bool {
 
-		tableName := key.(string)
+		if _, haveKey := rowsMap[tableName.(string)]; haveKey {
-
-		if _, haveKey := rowsMap[tableName]; haveKey {
 
 			seriesDims := value.([]*model2.ColumnWithType)
 			serDimsQuoted := make([]string, len(seriesDims))
@@ -84,7 +83,7 @@ func (this *ReportData2CK) Flush() (err error) {
 			}
 
 			bytesbuffer.WriteString("INSERT INTO ")
-			bytesbuffer.WriteString(tableName)
+			bytesbuffer.WriteString(tableName.(string))
 			bytesbuffer.WriteString(" (")
 			bytesbuffer.WriteString(strings.Join(serDimsQuoted, ","))
 			bytesbuffer.WriteString(") ")
@@ -92,22 +91,23 @@ func (this *ReportData2CK) Flush() (err error) {
 			bytesbuffer.WriteString(strings.Join(params, ","))
 			bytesbuffer.WriteString(")")
 
-			insertSql := bytesbuffer.String()
+			defer func() {
 				bytesbuffer.Reset()
+			}()
+
 			tx, err := db.ClickHouseSqlx.Begin()
 			if err != nil {
 				logs.Logger.Error("CK入库失败", zap.Error(err))
 				return false
 			}
-			stmt, err := tx.Prepare(insertSql)
+			stmt, err := tx.Prepare(bytesbuffer.String())
 			if err != nil {
 				logs.Logger.Error("CK入库失败", zap.Error(err))
 				return false
 			}
 			defer stmt.Close()
 			haveFail := false
-			for _, row := range rowsMap[tableName] {
+			for _, row := range rowsMap[tableName.(string)] {
-				logs.Logger.Sugar().Infof("insertSQL", insertSql,row)
 				if _, err := stmt.Exec(row...); err != nil {
 					logs.Logger.Error("CK入库失败", zap.Error(err))
 					haveFail = true
@@ -118,9 +118,8 @@ func (this *ReportData2CK) Flush() (err error) {
 					logs.Logger.Error("CK入库失败", zap.Error(err))
 					return false
 				} else {
-					lostTime := time.Now().Sub(startNow).String()
 					len := len(this.buffer)
-					logs.Logger.Info("CK入库成功，", zap.String("所花时间", lostTime), zap.Int("数据长度为", len))
+					logs.Logger.Info("CK入库成功，", zap.String("所花时间", time.Now().Sub(startNow).String()), zap.Int("数据长度为", len))
 				}
 			}
 		}

platform-basic-libs/sinker/clickhouse.go

@@ -84,13 +84,15 @@ func ClearDimsCacheByKey(key string){
 	dimsCacheMap.Delete(key)
 }
 
-func GetDims(database, table string, excludedColumns []string, conn *sqlx.DB) (dims []*model2.ColumnWithType, err error) {
+func GetDims(database, table string, excludedColumns []string, conn *sqlx.DB,onlyRedis bool) (dims []*model2.ColumnWithType, err error) {
-	dimsCachekey := GetDimsCachekey(database, table)
-	cache,load := dimsCacheMap.Load(dimsCachekey)
 
+	dimsCachekey := GetDimsCachekey(database, table)
+	if !onlyRedis{
+		cache,load := dimsCacheMap.Load(dimsCachekey)
 		if load {
 			return cache.([]*model2.ColumnWithType),nil
 		}
+	}
 
 	var json = jsoniter.ConfigCompatibleWithStandardLibrary