package smdl import ( "admin/apps/game/domain/entity" dto2 "admin/internal/model/dto" "admin/lib/xlog" "encoding/base64" "encoding/json" "fmt" "strconv" "strings" "time" ) type GameLogHook struct { } var removeFields = map[string]struct{}{ "pub_mediaid": {}, "pub_role_name": {}, "pub_udid": {}, } var fieldsAlias = map[string]string{ "pub_viplev": "vip等级", "pub_userid": "账号", "pub_totalcash": "总充值金额", "pub_serverid": "服务器id", "pub_rolename": "角色名", "pub_roleid": "角色id", "pub_lev": "等级", "pub_language": "语言", "pub_ip": "IP", } var base64decodeFields = map[string]struct{}{ "pub_rolename": {}, "chatlog_msg": {}, } var reasonValueAlias = map[string]map[string]string{ "gainitem_itempath": reasonGainDesc, "loseitem_itempath": reasonCostDesc, "addcoin_reasonstr": reasonGainDesc, "costcoin_reasonstr": reasonCostDesc, } var reasonEnumValueAlias = map[string]map[int]string{ "addcoin_coinpath": reasonCostEnumDesc, "costcoin_coinpath": reasonCostEnumDesc, } func init() { for i := 0; i < 15; i++ { removeFields["chatlog_score"] = struct{}{} removeFields["chatlog_score"+strconv.Itoa(i)] = struct{}{} removeFields["chatlog_label"] = struct{}{} removeFields["chatlog_label"+strconv.Itoa(i)] = struct{}{} removeFields["chatlog_checklevel"] = struct{}{} removeFields["chatlog_checklevel"+strconv.Itoa(i)] = struct{}{} } removeFields["chatlog_code"] = struct{}{} removeFields["chatlog_checkresult"] = struct{}{} } func (hook *GameLogHook) Trim(projectInfo *entity.Project, eventName []string, totalCount int, fieldsDescInfo []*dto2.GameLogFieldInfo, rows [][]any) ( int, []*dto2.GameLogFieldInfo, [][]any) { totalCount, fieldsDescInfo, rows = hook.getTrimHandlerByEventName(eventName)(projectInfo, totalCount, fieldsDescInfo, rows) return totalCount, fieldsDescInfo, rows } func (hook *GameLogHook) getTrimHandlerByEventName(eventName []string) func(projectInfo *entity.Project, totalCount int, fieldsDescInfo []*dto2.GameLogFieldInfo, rows [][]any) ( int, []*dto2.GameLogFieldInfo, [][]any) { for _, v := range eventName { if v == "chatlog" { return hook.trimChat } if v == "addcoin" || v == "costcoin" { return hook.trimCurrencyChange } if v == "gainitem" || v == "loseitem" { return hook.trimItemChange } } return nil } func (hook *GameLogHook) trimChat(projectInfo *entity.Project, totalCount int, fieldsDescInfo []*dto2.GameLogFieldInfo, rows [][]any) ( int, []*dto2.GameLogFieldInfo, [][]any) { newFields, newRows := hook.trimCommon(projectInfo, fieldsDescInfo, rows) extractFields := [][2]string{{"chatlog_channel", "渠道"}, {"chatlog_msg", "聊天内容"}} for _, eField := range extractFields { for colI, field := range fieldsDescInfo { if field.Name == eField[0] { field.Alias = eField[1] newFields = append(newFields, field) for rowI, row := range rows { if field.Name == "chatlog_msg" { newValue, err := base64.StdEncoding.DecodeString(row[colI].(string)) if err != nil { xlog.Warnf("base64 decode field:%v value:%v error:%v", field.Name, row[colI], err) } else { newRows[rowI] = append(newRows[rowI], string(newValue)) } } else if field.Name == "chatlog_channel" { channel := int(row[colI].(float64)) channelDesc, find := chatChannels[channel] if find { newRows[rowI] = append(newRows[rowI], channelDesc) } else { newRows[rowI] = append(newRows[rowI], fmt.Sprintf("%v", row[colI])) } } } break } } } return totalCount, newFields, newRows } func (hook *GameLogHook) trimCurrencyChange(projectInfo *entity.Project, totalCount int, fieldsDescInfo []*dto2.GameLogFieldInfo, rows [][]any) ( int, []*dto2.GameLogFieldInfo, [][]any) { newFields, newRows := hook.trimCommon(projectInfo, fieldsDescInfo, rows) newFields = append(newFields, []*dto2.GameLogFieldInfo{ {Name: "coinType", Alias: "货币类型"}, {Name: "coinPath", Alias: "途径"}, {Name: "beforeNum", Alias: "改变前数量"}, {Name: "deltaNum", Alias: "改变数量"}, {Name: "afterNum", Alias: "改变后数量"}, }...) extractFields := map[string][]string{ "addcoin": {"addcoin_cointype", "addcoin_coinpath", "addcoin_coin", "addcoin_coinleft"}, "costcoin": {"costcoin_cointype", "costcoin_coinpath", "costcoin_coin", "costcoin_coinleft"}, } // 刷新一下缓存 new(Items).GetItems(projectInfo) for rowI, row := range rows { eventName := row[0].(string) for _, eFieldName := range extractFields[eventName] { for fieldI, field := range fieldsDescInfo { if field.Name == eFieldName { colValue := row[fieldI] if field.Name == "addcoin_cointype" || field.Name == "costcoin_cointype" { // 货币类型要获取对应中文描述 coinType := int(colValue.(float64)) coinDesc := currencyDescTypes[coinType] colValue = coinDesc } else if field.Name == "addcoin_coinpath" { desc, find := reasonGainEnumDesc[int(colValue.(float64))] if find { colValue = desc } } else if field.Name == "costcoin_coinpath" { desc, find := reasonCostEnumDesc[int(colValue.(float64))] if find { colValue = desc } } newRows[rowI] = append(newRows[rowI], colValue) break } } } } // 计算一下改变前数量 for i, row := range newRows { deltaNum := row[len(row)-2].(float64) leftCount := row[len(row)-1].(float64) if row[0].(string) == "costcoin" { deltaNum = -deltaNum } beforeNum := leftCount - deltaNum row = row[:len(row)-2] row = append(row, beforeNum) row = append(row, deltaNum) row = append(row, leftCount) newRows[i] = row } return totalCount, newFields, newRows } func (hook *GameLogHook) trimItemChange(projectInfo *entity.Project, totalCount int, fieldsDescInfo []*dto2.GameLogFieldInfo, rows [][]any) ( int, []*dto2.GameLogFieldInfo, [][]any) { newFields, newRows := hook.trimCommon(projectInfo, fieldsDescInfo, rows) newFields = append(newFields, []*dto2.GameLogFieldInfo{ {Name: "itemid", Alias: "道具id"}, {Name: "itemName", Alias: "道具名称"}, {Name: "reason", Alias: "途径"}, {Name: "beforeNum", Alias: "改变前数量"}, {Name: "deltaNum", Alias: "改变数量"}, {Name: "afterNum", Alias: "改变后数量"}, }...) extractFields := map[string][]string{ "gainitem": {"gainitem_itemid", "gainitem_itempath", "gainitem_itemcount", "gainitem_itemleft"}, "loseitem": {"loseitem_itemid", "loseitem_itempath", "loseitem_itemcount", "loseitem_itemleft"}, } // 刷新一下缓存 new(Items).GetItems(projectInfo) for rowI, row := range rows { eventName := row[0].(string) for _, eFieldName := range extractFields[eventName] { for fieldI, field := range fieldsDescInfo { if field.Name == eFieldName { colValue := row[fieldI] if field.Name == "gainitem_itemid" || field.Name == "loseitem_itemid" { // 道具id要获取对应道具名 itemId := int(colValue.(float64)) if itemsCache != nil { itemInfo, find := itemsCache.mapItems[itemId] if find { itemName := itemInfo.Desc newRows[rowI] = append(newRows[rowI], itemId) newRows[rowI] = append(newRows[rowI], itemName) } else { newRows[rowI] = append(newRows[rowI], itemId) newRows[rowI] = append(newRows[rowI], strconv.Itoa(itemId)) } } else { newRows[rowI] = append(newRows[rowI], itemId) newRows[rowI] = append(newRows[rowI], strconv.Itoa(itemId)) } } else if field.Name == "gainitem_itempath" { desc, find := reasonGainDesc[colValue.(string)] if find { newRows[rowI] = append(newRows[rowI], desc) } else { newRows[rowI] = append(newRows[rowI], colValue) } } else if field.Name == "loseitem_itempath" { desc, find := reasonCostDesc[colValue.(string)] if find { newRows[rowI] = append(newRows[rowI], desc) } else { newRows[rowI] = append(newRows[rowI], colValue) } } else { newRows[rowI] = append(newRows[rowI], colValue) } break } } } } // 计算一下改变前数量 for i, row := range newRows { deltaNum := row[len(row)-2].(float64) leftCount := row[len(row)-1].(float64) if row[0].(string) == "loseitem" { deltaNum = -deltaNum } beforeNum := leftCount - deltaNum row = row[:len(row)-2] row = append(row, beforeNum) row = append(row, deltaNum) row = append(row, leftCount) newRows[i] = row } return totalCount, newFields, newRows } var commonExtractFields = [][2]string{ {"xwl_part_event", "事件类型"}, {"xwl_part_date", "时间"}, {"pub_userid", "账号"}, {"pub_roleid", "角色id"}, {"pub_rolename", "角色名"}, } func (hook *GameLogHook) trimCommon(projectInfo *entity.Project, fieldsDescInfo []*dto2.GameLogFieldInfo, rows [][]any) ( []*dto2.GameLogFieldInfo, [][]any) { newFields := make([]*dto2.GameLogFieldInfo, 0, 10) newRows := make([][]any, len(rows)) for _, eField := range commonExtractFields { for i, field := range fieldsDescInfo { if eField[0] == field.Name { field.Alias = eField[1] newFields = append(newFields, field) for j, newRow := range newRows { if field.Name == "pub_rolename" { newValue, err := base64.StdEncoding.DecodeString(rows[j][i].(string)) if err != nil { xlog.Warnf("base64 decode field:%v value:%v error:%v", field.Name, rows[j][i], err) } else { newRows[j] = append(newRows[j], string(newValue)) } } else if field.Name == "xwl_part_date" { v := rows[j][i].(time.Time) newRows[j] = append(newRow, v.Format(time.DateTime)) } else { newRows[j] = append(newRow, rows[j][i]) } } break } } } return newFields, newRows } func (hook *GameLogHook) TrimBak(projectInfo *entity.Project, eventName []string, totalCount int, fieldsDescInfo []*dto2.GameLogFieldInfo, rows [][]any) ( int, []*dto2.GameLogFieldInfo, [][]any) { // 删除不需要的字段 i := 0 for { if i >= len(fieldsDescInfo) { break } field := fieldsDescInfo[i] if _, find := removeFields[field.Name]; find { // 找到这个字段在去掉的列表里 fieldsDescInfo = append(fieldsDescInfo[:i], fieldsDescInfo[i+1:]...) // 对应所有数据行里也删除这个值 for rowI, row := range rows { row = append(row[:i], row[i+1:]...) rows[rowI] = row } } else { // 查找要不要把value做base64解码 if _, find := base64decodeFields[field.Name]; find { for _, row := range rows { newValue, err := base64.StdEncoding.DecodeString(row[i].(string)) if err != nil { xlog.Warnf("base64 decode field:%v value:%v error:%v", field.Name, row[i], err) } else { row[i] = string(newValue) } } } i++ } } //fBin, _ := json.Marshal(&fieldsDescInfo) //rBin, _ := json.Marshal(&rows) //xlog.Tracef("gamelog1 query result:%v, rows:%v", string(fBin), string(rBin)) for i, f := range fieldsDescInfo { // 修改每一行日志值描述 if replaceMap, findReplaceValue := reasonValueAlias[f.Name]; findReplaceValue { for j := range rows { curValue, ok := rows[j][i].(string) if ok { newValue, find := replaceMap[curValue] if find { rows[j][i] = newValue } } } } else if replaceMap, findReplaceValue := reasonEnumValueAlias[f.Name]; findReplaceValue { for j := range rows { curValue, ok := rows[j][i].(float64) if ok { newValue, find := replaceMap[int(curValue)] if find { rows[j][i] = newValue } } } } } fieldsDescInfo, rows = (&queryResultInfo{fields: fieldsDescInfo, rows: rows}).tidyByEventDescInfo(eventName) //fBin, _ = json.Marshal(&fieldsDescInfo) //rBin, _ = json.Marshal(&rows) //xlog.Tracef("gamelog1 query result:%v, rows:%v", string(fBin), string(rBin)) for i, f := range fieldsDescInfo { // 修改每一行日志别名 if f.Name == "xwl_part_event" { for j := range rows { opEventName := rows[j][i].(string) descInfo, find := globEventList[opEventName] if find { rows[j][i] = descInfo.Alias } } } else if f.Name == "xwl_part_date" { for j := range rows { eventDate := rows[j][i].(time.Time) //eventDate, err := time.Parse("2006-01-02T15:04:05+07:00", eventDateString) rows[j][i] = eventDate.Format(time.DateTime) } } // 修改每一行公共属性别名 alias, find := fieldsAlias[f.Name] if find && f.Name == f.Alias { f.Alias = alias } } fBin, _ := json.Marshal(&fieldsDescInfo) rBin, _ := json.Marshal(&rows) xlog.Tracef("gamelog2 query result:%v, rows:%v", string(fBin), string(rBin)) i = 0 for { if i >= len(fieldsDescInfo) { break } f := fieldsDescInfo[i] swapI := 2 if f.Name == "pub_serverid" || f.Name == "pub_roleid" || f.Name == "pub_rolename" { pubF := fieldsDescInfo[i] for moveI := i; moveI > swapI; moveI-- { fieldsDescInfo[moveI] = fieldsDescInfo[moveI-1] } fieldsDescInfo[swapI] = pubF for _, row := range rows { pubValue := row[i] for moveI := i; moveI > swapI; moveI-- { row[moveI] = row[moveI-1] } row[swapI] = pubValue } swapI++ } i++ } return totalCount, fieldsDescInfo, rows } type queryResultInfo struct { fields []*dto2.GameLogFieldInfo rows [][]any } func (info *queryResultInfo) tidyByEventDescInfo(eventNameList []string) ([]*dto2.GameLogFieldInfo, [][]any) { if len(eventNameList) == 0 { return info.fields, info.rows } if len(eventNameList) == 1 { en := eventNameList[0] enDescInfo, find := globEventList[en] if !find { panic(en) } for _, f := range info.fields { for _, df := range enDescInfo.fields { fName := removeFieldEventName(en, f.Name) if fName == df.Name && f.Name == f.Alias { f.Alias = df.Alias break } } } return info.fields, info.rows } // 神魔的埋点里字段名是日志名_字段名,所以如果一次游戏日志查询包含多个事件名,会查出稀疏数据, // 例如gainitem,loseitem两个日志,都会存在gainitem_itemid,loseitem_itemid,需要把对应 // 为0的稀疏字段去掉 // 具体方案: // 1.创建新的行数据,将自己日志的字段值先追加进来,然后前面补上日志名,后面补上公共字段, // 2.字段描述信息直接全部去重 newRows := make([][]any, len(info.rows)) for rowI, row := range info.rows { curRowEventName := row[0].(string) enDescInfo, find := globEventList[curRowEventName] if !find { // 不存在描述信息,先崩溃下 panic(curRowEventName) } for _, fieldDesc := range enDescInfo.fields { for i, field := range info.fields { fName := removeFieldEventName(curRowEventName, field.Name) if fName == fieldDesc.Name { newRows[rowI] = append(newRows[rowI], row[i]) break } } } } // 上面步骤走完,newRows每一行都是各自日志的字段值了,但是每一行长度可能不一样,例如gainitem就比loseitem多一个字段 // 下面步骤会把字段描述信息通过事件名具有的所有字段去重追加,例如gainitem就是全量的字段,loseitem就是子集 newFieldsDescInfo := make([]*dto2.GameLogFieldInfo, 0, len(info.fields)) for _, en := range eventNameList { enDescInfo, find := globEventList[en] if !find { panic(en) } find = false for _, fd := range enDescInfo.fields { for _, newFd := range newFieldsDescInfo { if fd.Name == newFd.Name { find = true break } } if !find { // 追加 for _, oldFd := range info.fields { fName := removeFieldEventName(en, oldFd.Name) if fName == fd.Name { newFieldInfo := &dto2.GameLogFieldInfo{ Name: fName, Alias: oldFd.Alias, IsPublicField: oldFd.IsPublicField, FieldType: oldFd.FieldType, } if oldFd.Name == oldFd.Alias { newFieldInfo.Alias = fd.Alias } newFieldsDescInfo = append(newFieldsDescInfo, newFieldInfo) break } } } } } // 走完上面这一步,newFieldsDescInfo就是事件列表具有的所有去重字段了 allEventFieldsLen := len(newFieldsDescInfo) // 给newRows和newFieldsDescInfo前追加公共字段了 // 追加前面的日志名、时间 tmpFields := make([]*dto2.GameLogFieldInfo, 2) copy(tmpFields, info.fields[:2]) newFieldsDescInfo = append(tmpFields, newFieldsDescInfo...) for i, newRow := range newRows { if len(newRow) < allEventFieldsLen { rowCols := len(newRow) for j := 0; j < allEventFieldsLen-rowCols; j++ { // 补充这一行日志没有的字段 newRow = append(newRow, "\\") } } else if len(newRow) > allEventFieldsLen { panic(fmt.Sprintf("%v,%v", len(newRow), allEventFieldsLen)) } tmpCols := make([]any, 2) copy(tmpCols, info.rows[i][:2]) newRow = append(tmpCols, newRow...) newRows[i] = newRow } // 给newRows和newFieldsDescInfo后追加公共字段了 for i, f := range info.fields { if len(f.Name) >= 4 && f.Name[:4] == "pub_" { newFieldsDescInfo = append(newFieldsDescInfo, info.fields[i:]...) for rowI, newRow := range newRows { newRow = append(newRow, info.rows[rowI][i:]...) newRows[rowI] = newRow } break } } return newFieldsDescInfo, newRows } // 因为推送到clickhouse的字段名都是日志名_字段名的格式,放置不同日志出现同名字段冲突,这里直接去掉一下日志名 func removeFieldEventName(eventName, fieldName string) string { return strings.ReplaceAll(fieldName, eventName+"_", "") }