package smdl import ( "admin/apps/game/domain/entity" dto2 "admin/internal/model/dto" "admin/lib/xlog" "encoding/base64" "encoding/json" "fmt" "strconv" "strings" "time" ) type GameLogHook struct { } var removeFields = map[string]struct{}{ "pub_mediaid": {}, "pub_role_name": {}, "pub_udid": {}, } var fieldsAlias = map[string]string{ "pub_viplev": "vip等级", "pub_userid": "账号", "pub_totalcash": "总充值金额", "pub_serverid": "服务器id", "pub_rolename": "角色名", "pub_roleid": "角色id", "pub_lev": "等级", "pub_language": "语言", "pub_ip": "IP", } var base64decodeFields = map[string]struct{}{ "pub_rolename": {}, "chatlog_msg": {}, } func init() { for i := 0; i < 15; i++ { removeFields["chatlog_score"] = struct{}{} removeFields["chatlog_score"+strconv.Itoa(i)] = struct{}{} removeFields["chatlog_label"] = struct{}{} removeFields["chatlog_label"+strconv.Itoa(i)] = struct{}{} removeFields["chatlog_checklevel"] = struct{}{} removeFields["chatlog_checklevel"+strconv.Itoa(i)] = struct{}{} } removeFields["chatlog_code"] = struct{}{} removeFields["chatlog_checkresult"] = struct{}{} } func (hook *GameLogHook) Trim(projectInfo *entity.Project, eventName []string, totalCount int, fieldsDescInfo []*dto2.GameLogFieldInfo, rows [][]any) ( int, []*dto2.GameLogFieldInfo, [][]any) { // 删除不需要的字段 i := 0 for { if i >= len(fieldsDescInfo) { break } field := fieldsDescInfo[i] if _, find := removeFields[field.Name]; find { // 找到这个字段在去掉的列表里 fieldsDescInfo = append(fieldsDescInfo[:i], fieldsDescInfo[i+1:]...) // 对应所有数据行里也删除这个值 for rowI, row := range rows { row = append(row[:i], row[i+1:]...) rows[rowI] = row } } else { // 查找要不要把value做base64解码 if _, find := base64decodeFields[field.Name]; find { for _, row := range rows { newValue, err := base64.StdEncoding.DecodeString(row[i].(string)) if err != nil { xlog.Warnf("base64 decode field:%v value:%v error:%v", field.Name, row[i], err) } else { row[i] = string(newValue) } } } i++ } } //fBin, _ := json.Marshal(&fieldsDescInfo) //rBin, _ := json.Marshal(&rows) //xlog.Tracef("gamelog1 query result:%v, rows:%v", string(fBin), string(rBin)) fieldsDescInfo, rows = (&queryResultInfo{fields: fieldsDescInfo, rows: rows}).tidyByEventDescInfo(eventName) //fBin, _ = json.Marshal(&fieldsDescInfo) //rBin, _ = json.Marshal(&rows) //xlog.Tracef("gamelog1 query result:%v, rows:%v", string(fBin), string(rBin)) for i, f := range fieldsDescInfo { // 修改每一行日志别名 if f.Name == "xwl_part_event" { for j := range rows { opEventName := rows[j][i].(string) descInfo, find := globEventList[opEventName] if find { rows[j][i] = descInfo.Alias } } } else if f.Name == "xwl_part_date" { for j := range rows { eventDate := rows[j][i].(time.Time) //eventDate, err := time.Parse("2006-01-02T15:04:05+07:00", eventDateString) rows[j][i] = eventDate.Format(time.DateTime) } } // 修改每一行公共属性别名 alias, find := fieldsAlias[f.Name] if find && f.Name == f.Alias { f.Alias = alias } } fBin, _ := json.Marshal(&fieldsDescInfo) rBin, _ := json.Marshal(&rows) xlog.Tracef("gamelog2 query result:%v, rows:%v", string(fBin), string(rBin)) i = 0 for { if i >= len(fieldsDescInfo) { break } f := fieldsDescInfo[i] swapI := 2 if f.Name == "pub_serverid" || f.Name == "pub_roleid" || f.Name == "pub_rolename" { pubF := fieldsDescInfo[i] for moveI := i; moveI > swapI; moveI-- { fieldsDescInfo[moveI] = fieldsDescInfo[moveI-1] } fieldsDescInfo[swapI] = pubF for _, row := range rows { pubValue := row[i] for moveI := i; moveI > swapI; moveI-- { row[moveI] = row[moveI-1] } row[swapI] = pubValue } swapI++ } i++ } return totalCount, fieldsDescInfo, rows } type queryResultInfo struct { fields []*dto2.GameLogFieldInfo rows [][]any } func (info *queryResultInfo) tidyByEventDescInfo(eventNameList []string) ([]*dto2.GameLogFieldInfo, [][]any) { if len(eventNameList) == 0 { return info.fields, info.rows } if len(eventNameList) == 1 { en := eventNameList[0] enDescInfo, find := globEventList[en] if !find { panic(en) } for _, f := range info.fields { for _, df := range enDescInfo.fields { fName := removeFieldEventName(en, f.Name) if fName == df.Name && f.Name == f.Alias { f.Alias = df.Alias break } } } return info.fields, info.rows } // 神魔的埋点里字段名是日志名_字段名,所以如果一次游戏日志查询包含多个事件名,会查出稀疏数据, // 例如gainitem,loseitem两个日志,都会存在gainitem_itemid,loseitem_itemid,需要把对应 // 为0的稀疏字段去掉 // 具体方案: // 1.创建新的行数据,将自己日志的字段值先追加进来,然后前面补上日志名,后面补上公共字段, // 2.字段描述信息直接全部去重 newRows := make([][]any, len(info.rows)) for rowI, row := range info.rows { curRowEventName := row[0].(string) enDescInfo, find := globEventList[curRowEventName] if !find { // 不存在描述信息,先崩溃下 panic(curRowEventName) } for _, fieldDesc := range enDescInfo.fields { for i, field := range info.fields { fName := removeFieldEventName(curRowEventName, field.Name) if fName == fieldDesc.Name { newRows[rowI] = append(newRows[rowI], row[i]) break } } } } // 上面步骤走完,newRows每一行都是各自日志的字段值了,但是每一行长度可能不一样,例如gainitem就比loseitem多一个字段 // 下面步骤会把字段描述信息通过事件名具有的所有字段去重追加,例如gainitem就是全量的字段,loseitem就是子集 newFieldsDescInfo := make([]*dto2.GameLogFieldInfo, 0, len(info.fields)) for _, en := range eventNameList { enDescInfo, find := globEventList[en] if !find { panic(en) } find = false for _, fd := range enDescInfo.fields { for _, newFd := range newFieldsDescInfo { if fd.Name == newFd.Name { find = true break } } if !find { // 追加 for _, oldFd := range info.fields { fName := removeFieldEventName(en, oldFd.Name) if fName == fd.Name { newFieldInfo := &dto2.GameLogFieldInfo{ Name: fName, Alias: oldFd.Alias, IsPublicField: oldFd.IsPublicField, FieldType: oldFd.FieldType, } if oldFd.Name == oldFd.Alias { newFieldInfo.Alias = fd.Alias } newFieldsDescInfo = append(newFieldsDescInfo, newFieldInfo) break } } } } } // 走完上面这一步,newFieldsDescInfo就是事件列表具有的所有去重字段了 allEventFieldsLen := len(newFieldsDescInfo) // 给newRows和newFieldsDescInfo前追加公共字段了 // 追加前面的日志名、时间 tmpFields := make([]*dto2.GameLogFieldInfo, 2) copy(tmpFields, info.fields[:2]) newFieldsDescInfo = append(tmpFields, newFieldsDescInfo...) for i, newRow := range newRows { if len(newRow) < allEventFieldsLen { rowCols := len(newRow) for j := 0; j < allEventFieldsLen-rowCols; j++ { // 补充这一行日志没有的字段 newRow = append(newRow, "\\") } } else if len(newRow) > allEventFieldsLen { panic(fmt.Sprintf("%v,%v", len(newRow), allEventFieldsLen)) } tmpCols := make([]any, 2) copy(tmpCols, info.rows[i][:2]) newRow = append(tmpCols, newRow...) newRows[i] = newRow } // 给newRows和newFieldsDescInfo后追加公共字段了 for i, f := range info.fields { if len(f.Name) >= 4 && f.Name[:4] == "pub_" { newFieldsDescInfo = append(newFieldsDescInfo, info.fields[i:]...) for rowI, newRow := range newRows { newRow = append(newRow, info.rows[rowI][i:]...) newRows[rowI] = newRow } break } } return newFieldsDescInfo, newRows } // 因为推送到clickhouse的字段名都是日志名_字段名的格式,放置不同日志出现同名字段冲突,这里直接去掉一下日志名 func removeFieldEventName(eventName, fieldName string) string { return strings.ReplaceAll(fieldName, eventName+"_", "") }