uniugm/admin/apps/game/domain/projects/smdl/gamelog.go

package smdl

import (
	"admin/apps/game/domain/entity"
	dto2 "admin/internal/model/dto"
	"admin/lib/xlog"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"strconv"
	"strings"
	"time"
)

type GameLogHook struct {
}

var removeFields = map[string]struct{}{
	"pub_mediaid":   {},
	"pub_role_name": {},
	"pub_udid":      {},
}

var fieldsAlias = map[string]string{
	"pub_viplev":    "vip等级",
	"pub_userid":    "账号",
	"pub_totalcash": "总充值金额",
	"pub_serverid":  "服务器id",
	"pub_rolename":  "角色名",
	"pub_roleid":    "角色id",
	"pub_lev":       "等级",
	"pub_language":  "语言",
	"pub_ip":        "IP",
}

var base64decodeFields = map[string]struct{}{
	"pub_rolename": {},
	"chatlog_msg":  {},
}

func init() {
	for i := 0; i < 15; i++ {
		removeFields["chatlog_score"] = struct{}{}
		removeFields["chatlog_score"+strconv.Itoa(i)] = struct{}{}
		removeFields["chatlog_label"] = struct{}{}
		removeFields["chatlog_label"+strconv.Itoa(i)] = struct{}{}
		removeFields["chatlog_checklevel"] = struct{}{}
		removeFields["chatlog_checklevel"+strconv.Itoa(i)] = struct{}{}
	}
	removeFields["chatlog_code"] = struct{}{}
	removeFields["chatlog_checkresult"] = struct{}{}
}

func (hook *GameLogHook) Trim(projectInfo *entity.Project, eventName []string, totalCount int, fieldsDescInfo []*dto2.GameLogFieldInfo, rows [][]any) (
	int, []*dto2.GameLogFieldInfo, [][]any) {

	// 删除不需要的字段
	i := 0
	for {
		if i >= len(fieldsDescInfo) {
			break
		}
		field := fieldsDescInfo[i]
		if _, find := removeFields[field.Name]; find {
			// 找到这个字段在去掉的列表里
			fieldsDescInfo = append(fieldsDescInfo[:i], fieldsDescInfo[i+1:]...)
			// 对应所有数据行里也删除这个值
			for rowI, row := range rows {
				row = append(row[:i], row[i+1:]...)
				rows[rowI] = row
			}
		} else {
			// 查找要不要把value做base64解码
			if _, find := base64decodeFields[field.Name]; find {
				for _, row := range rows {
					newValue, err := base64.StdEncoding.DecodeString(row[i].(string))
					if err != nil {
						xlog.Warnf("base64 decode field:%v value:%v error:%v", field.Name, row[i], err)
					} else {
						row[i] = string(newValue)
					}
				}
			}
			i++
		}
	}

	//fBin, _ := json.Marshal(&fieldsDescInfo)
	//rBin, _ := json.Marshal(&rows)
	//xlog.Tracef("gamelog1 query result:%v, rows:%v", string(fBin), string(rBin))

	fieldsDescInfo, rows = (&queryResultInfo{fields: fieldsDescInfo, rows: rows}).tidyByEventDescInfo(eventName)

	//fBin, _ = json.Marshal(&fieldsDescInfo)
	//rBin, _ = json.Marshal(&rows)
	//xlog.Tracef("gamelog1 query result:%v, rows:%v", string(fBin), string(rBin))

	for i, f := range fieldsDescInfo {
		// 修改每一行日志别名
		if f.Name == "xwl_part_event" {
			for j := range rows {
				opEventName := rows[j][i].(string)
				descInfo, find := globEventList[opEventName]
				if find {
					rows[j][i] = descInfo.Alias
				}
			}
		} else if f.Name == "xwl_part_date" {
			for j := range rows {
				eventDate := rows[j][i].(time.Time)
				//eventDate, err := time.Parse("2006-01-02T15:04:05+07:00", eventDateString)
				rows[j][i] = eventDate.Format(time.DateTime)
			}
		}
		// 修改每一行公共属性别名
		alias, find := fieldsAlias[f.Name]
		if find && f.Name == f.Alias {
			f.Alias = alias
		}
	}

	fBin, _ := json.Marshal(&fieldsDescInfo)
	rBin, _ := json.Marshal(&rows)
	xlog.Tracef("gamelog2 query result:%v, rows:%v", string(fBin), string(rBin))

	i = 0
	for {
		if i >= len(fieldsDescInfo) {
			break
		}
		f := fieldsDescInfo[i]
		swapI := 2
		if f.Name == "pub_serverid" || f.Name == "pub_roleid" || f.Name == "pub_rolename" {
			pubF := fieldsDescInfo[i]
			for moveI := i; moveI > swapI; moveI-- {
				fieldsDescInfo[moveI] = fieldsDescInfo[moveI-1]
			}
			fieldsDescInfo[swapI] = pubF

			for _, row := range rows {
				pubValue := row[i]
				for moveI := i; moveI > swapI; moveI-- {
					row[moveI] = row[moveI-1]
				}
				row[swapI] = pubValue
			}
			swapI++
		}
		i++
	}

	return totalCount, fieldsDescInfo, rows
}

type queryResultInfo struct {
	fields []*dto2.GameLogFieldInfo
	rows   [][]any
}

func (info *queryResultInfo) tidyByEventDescInfo(eventNameList []string) ([]*dto2.GameLogFieldInfo, [][]any) {
	if len(eventNameList) == 0 {
		return info.fields, info.rows
	}
	if len(eventNameList) == 1 {
		en := eventNameList[0]
		enDescInfo, find := globEventList[en]
		if !find {
			panic(en)
		}
		for _, f := range info.fields {
			for _, df := range enDescInfo.fields {
				fName := removeFieldEventName(en, f.Name)
				if fName == df.Name && f.Name == f.Alias {
					f.Alias = df.Alias
					break
				}
			}
		}
		return info.fields, info.rows
	}

	// 神魔的埋点里字段名是日志名_字段名，所以如果一次游戏日志查询包含多个事件名，会查出稀疏数据，
	// 例如gainitem,loseitem两个日志，都会存在gainitem_itemid,loseitem_itemid，需要把对应
	// 为0的稀疏字段去掉

	// 具体方案：
	// 1.创建新的行数据，将自己日志的字段值先追加进来，然后前面补上日志名，后面补上公共字段，
	// 2.字段描述信息直接全部去重

	newRows := make([][]any, len(info.rows))
	for rowI, row := range info.rows {
		curRowEventName := row[0].(string)
		enDescInfo, find := globEventList[curRowEventName]
		if !find {
			// 不存在描述信息，先崩溃下
			panic(curRowEventName)
		}

		for _, fieldDesc := range enDescInfo.fields {
			for i, field := range info.fields {
				fName := removeFieldEventName(curRowEventName, field.Name)
				if fName == fieldDesc.Name {
					newRows[rowI] = append(newRows[rowI], row[i])
					break
				}
			}
		}
	}

	// 上面步骤走完，newRows每一行都是各自日志的字段值了，但是每一行长度可能不一样，例如gainitem就比loseitem多一个字段
	// 下面步骤会把字段描述信息通过事件名具有的所有字段去重追加，例如gainitem就是全量的字段，loseitem就是子集
	newFieldsDescInfo := make([]*dto2.GameLogFieldInfo, 0, len(info.fields))
	for _, en := range eventNameList {
		enDescInfo, find := globEventList[en]
		if !find {
			panic(en)
		}
		find = false
		for _, fd := range enDescInfo.fields {
			for _, newFd := range newFieldsDescInfo {
				if fd.Name == newFd.Name {
					find = true
					break
				}
			}
			if !find {
				// 追加
				for _, oldFd := range info.fields {
					fName := removeFieldEventName(en, oldFd.Name)
					if fName == fd.Name {
						newFieldInfo := &dto2.GameLogFieldInfo{
							Name:          fName,
							Alias:         oldFd.Alias,
							IsPublicField: oldFd.IsPublicField,
							FieldType:     oldFd.FieldType,
						}
						if oldFd.Name == oldFd.Alias {
							newFieldInfo.Alias = fd.Alias
						}
						newFieldsDescInfo = append(newFieldsDescInfo, newFieldInfo)
						break
					}
				}
			}
		}
	}

	// 走完上面这一步，newFieldsDescInfo就是事件列表具有的所有去重字段了
	allEventFieldsLen := len(newFieldsDescInfo)

	// 给newRows和newFieldsDescInfo前追加公共字段了
	// 追加前面的日志名、时间
	tmpFields := make([]*dto2.GameLogFieldInfo, 2)
	copy(tmpFields, info.fields[:2])
	newFieldsDescInfo = append(tmpFields, newFieldsDescInfo...)
	for i, newRow := range newRows {
		if len(newRow) < allEventFieldsLen {
			rowCols := len(newRow)
			for j := 0; j < allEventFieldsLen-rowCols; j++ {
				// 补充这一行日志没有的字段
				newRow = append(newRow, "\\")
			}
		} else if len(newRow) > allEventFieldsLen {
			panic(fmt.Sprintf("%v,%v", len(newRow), allEventFieldsLen))
		}
		tmpCols := make([]any, 2)
		copy(tmpCols, info.rows[i][:2])
		newRow = append(tmpCols, newRow...)
		newRows[i] = newRow
	}

	// 给newRows和newFieldsDescInfo后追加公共字段了
	for i, f := range info.fields {
		if len(f.Name) >= 4 && f.Name[:4] == "pub_" {
			newFieldsDescInfo = append(newFieldsDescInfo, info.fields[i:]...)
			for rowI, newRow := range newRows {
				newRow = append(newRow, info.rows[rowI][i:]...)
				newRows[rowI] = newRow
			}
			break
		}
	}

	return newFieldsDescInfo, newRows
}

// 因为推送到clickhouse的字段名都是日志名_字段名的格式，放置不同日志出现同名字段冲突，这里直接去掉一下日志名
func removeFieldEventName(eventName, fieldName string) string {
	return strings.ReplaceAll(fieldName, eventName+"_", "")
}