291 lines
8.2 KiB
Go
291 lines
8.2 KiB
Go
package smdl
|
||
|
||
import (
|
||
"admin/apps/game/domain/entity"
|
||
dto2 "admin/internal/model/dto"
|
||
"admin/lib/xlog"
|
||
"encoding/base64"
|
||
"encoding/json"
|
||
"fmt"
|
||
"strconv"
|
||
"strings"
|
||
"time"
|
||
)
|
||
|
||
type GameLogHook struct {
|
||
}
|
||
|
||
var removeFields = map[string]struct{}{
|
||
"pub_mediaid": {},
|
||
"pub_role_name": {},
|
||
"pub_udid": {},
|
||
}
|
||
|
||
var fieldsAlias = map[string]string{
|
||
"pub_viplev": "vip等级",
|
||
"pub_userid": "账号",
|
||
"pub_totalcash": "总充值金额",
|
||
"pub_serverid": "服务器id",
|
||
"pub_rolename": "角色名",
|
||
"pub_roleid": "角色id",
|
||
"pub_lev": "等级",
|
||
"pub_language": "语言",
|
||
"pub_ip": "IP",
|
||
}
|
||
|
||
var base64decodeFields = map[string]struct{}{
|
||
"pub_rolename": {},
|
||
"chatlog_msg": {},
|
||
}
|
||
|
||
func init() {
|
||
for i := 0; i < 15; i++ {
|
||
removeFields["chatlog_score"] = struct{}{}
|
||
removeFields["chatlog_score"+strconv.Itoa(i)] = struct{}{}
|
||
removeFields["chatlog_label"] = struct{}{}
|
||
removeFields["chatlog_label"+strconv.Itoa(i)] = struct{}{}
|
||
removeFields["chatlog_checklevel"] = struct{}{}
|
||
removeFields["chatlog_checklevel"+strconv.Itoa(i)] = struct{}{}
|
||
}
|
||
removeFields["chatlog_code"] = struct{}{}
|
||
removeFields["chatlog_checkresult"] = struct{}{}
|
||
}
|
||
|
||
func (hook *GameLogHook) Trim(projectInfo *entity.Project, eventName []string, totalCount int, fieldsDescInfo []*dto2.GameLogFieldInfo, rows [][]any) (
|
||
int, []*dto2.GameLogFieldInfo, [][]any) {
|
||
|
||
// 删除不需要的字段
|
||
i := 0
|
||
for {
|
||
if i >= len(fieldsDescInfo) {
|
||
break
|
||
}
|
||
field := fieldsDescInfo[i]
|
||
if _, find := removeFields[field.Name]; find {
|
||
// 找到这个字段在去掉的列表里
|
||
fieldsDescInfo = append(fieldsDescInfo[:i], fieldsDescInfo[i+1:]...)
|
||
// 对应所有数据行里也删除这个值
|
||
for rowI, row := range rows {
|
||
row = append(row[:i], row[i+1:]...)
|
||
rows[rowI] = row
|
||
}
|
||
} else {
|
||
// 查找要不要把value做base64解码
|
||
if _, find := base64decodeFields[field.Name]; find {
|
||
for _, row := range rows {
|
||
newValue, err := base64.StdEncoding.DecodeString(row[i].(string))
|
||
if err != nil {
|
||
xlog.Warnf("base64 decode field:%v value:%v error:%v", field.Name, row[i], err)
|
||
} else {
|
||
row[i] = string(newValue)
|
||
}
|
||
}
|
||
}
|
||
i++
|
||
}
|
||
}
|
||
|
||
//fBin, _ := json.Marshal(&fieldsDescInfo)
|
||
//rBin, _ := json.Marshal(&rows)
|
||
//xlog.Tracef("gamelog1 query result:%v, rows:%v", string(fBin), string(rBin))
|
||
|
||
fieldsDescInfo, rows = (&queryResultInfo{fields: fieldsDescInfo, rows: rows}).tidyByEventDescInfo(eventName)
|
||
|
||
//fBin, _ = json.Marshal(&fieldsDescInfo)
|
||
//rBin, _ = json.Marshal(&rows)
|
||
//xlog.Tracef("gamelog1 query result:%v, rows:%v", string(fBin), string(rBin))
|
||
|
||
for i, f := range fieldsDescInfo {
|
||
// 修改每一行日志别名
|
||
if f.Name == "xwl_part_event" {
|
||
for j := range rows {
|
||
opEventName := rows[j][i].(string)
|
||
descInfo, find := globEventList[opEventName]
|
||
if find {
|
||
rows[j][i] = descInfo.Alias
|
||
}
|
||
}
|
||
} else if f.Name == "xwl_part_date" {
|
||
for j := range rows {
|
||
eventDate := rows[j][i].(time.Time)
|
||
//eventDate, err := time.Parse("2006-01-02T15:04:05+07:00", eventDateString)
|
||
rows[j][i] = eventDate.Format(time.DateTime)
|
||
}
|
||
}
|
||
// 修改每一行公共属性别名
|
||
alias, find := fieldsAlias[f.Name]
|
||
if find && f.Name == f.Alias {
|
||
f.Alias = alias
|
||
}
|
||
}
|
||
|
||
fBin, _ := json.Marshal(&fieldsDescInfo)
|
||
rBin, _ := json.Marshal(&rows)
|
||
xlog.Tracef("gamelog2 query result:%v, rows:%v", string(fBin), string(rBin))
|
||
|
||
i = 0
|
||
for {
|
||
if i >= len(fieldsDescInfo) {
|
||
break
|
||
}
|
||
f := fieldsDescInfo[i]
|
||
swapI := 2
|
||
if f.Name == "pub_serverid" || f.Name == "pub_roleid" || f.Name == "pub_rolename" {
|
||
pubF := fieldsDescInfo[i]
|
||
for moveI := i; moveI > swapI; moveI-- {
|
||
fieldsDescInfo[moveI] = fieldsDescInfo[moveI-1]
|
||
}
|
||
fieldsDescInfo[swapI] = pubF
|
||
|
||
for _, row := range rows {
|
||
pubValue := row[i]
|
||
for moveI := i; moveI > swapI; moveI-- {
|
||
row[moveI] = row[moveI-1]
|
||
}
|
||
row[swapI] = pubValue
|
||
}
|
||
swapI++
|
||
}
|
||
i++
|
||
}
|
||
|
||
return totalCount, fieldsDescInfo, rows
|
||
}
|
||
|
||
type queryResultInfo struct {
|
||
fields []*dto2.GameLogFieldInfo
|
||
rows [][]any
|
||
}
|
||
|
||
func (info *queryResultInfo) tidyByEventDescInfo(eventNameList []string) ([]*dto2.GameLogFieldInfo, [][]any) {
|
||
if len(eventNameList) == 0 {
|
||
return info.fields, info.rows
|
||
}
|
||
if len(eventNameList) == 1 {
|
||
en := eventNameList[0]
|
||
enDescInfo, find := globEventList[en]
|
||
if !find {
|
||
panic(en)
|
||
}
|
||
for _, f := range info.fields {
|
||
for _, df := range enDescInfo.fields {
|
||
fName := removeFieldEventName(en, f.Name)
|
||
if fName == df.Name && f.Name == f.Alias {
|
||
f.Alias = df.Alias
|
||
break
|
||
}
|
||
}
|
||
}
|
||
return info.fields, info.rows
|
||
}
|
||
|
||
// 神魔的埋点里字段名是日志名_字段名，所以如果一次游戏日志查询包含多个事件名，会查出稀疏数据，
|
||
// 例如gainitem,loseitem两个日志，都会存在gainitem_itemid,loseitem_itemid，需要把对应
|
||
// 为0的稀疏字段去掉
|
||
|
||
// 具体方案：
|
||
// 1.创建新的行数据，将自己日志的字段值先追加进来，然后前面补上日志名，后面补上公共字段，
|
||
// 2.字段描述信息直接全部去重
|
||
|
||
newRows := make([][]any, len(info.rows))
|
||
for rowI, row := range info.rows {
|
||
curRowEventName := row[0].(string)
|
||
enDescInfo, find := globEventList[curRowEventName]
|
||
if !find {
|
||
// 不存在描述信息，先崩溃下
|
||
panic(curRowEventName)
|
||
}
|
||
|
||
for _, fieldDesc := range enDescInfo.fields {
|
||
for i, field := range info.fields {
|
||
fName := removeFieldEventName(curRowEventName, field.Name)
|
||
if fName == fieldDesc.Name {
|
||
newRows[rowI] = append(newRows[rowI], row[i])
|
||
break
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// 上面步骤走完，newRows每一行都是各自日志的字段值了，但是每一行长度可能不一样，例如gainitem就比loseitem多一个字段
|
||
// 下面步骤会把字段描述信息通过事件名具有的所有字段去重追加，例如gainitem就是全量的字段，loseitem就是子集
|
||
newFieldsDescInfo := make([]*dto2.GameLogFieldInfo, 0, len(info.fields))
|
||
for _, en := range eventNameList {
|
||
enDescInfo, find := globEventList[en]
|
||
if !find {
|
||
panic(en)
|
||
}
|
||
find = false
|
||
for _, fd := range enDescInfo.fields {
|
||
for _, newFd := range newFieldsDescInfo {
|
||
if fd.Name == newFd.Name {
|
||
find = true
|
||
break
|
||
}
|
||
}
|
||
if !find {
|
||
// 追加
|
||
for _, oldFd := range info.fields {
|
||
fName := removeFieldEventName(en, oldFd.Name)
|
||
if fName == fd.Name {
|
||
newFieldInfo := &dto2.GameLogFieldInfo{
|
||
Name: fName,
|
||
Alias: oldFd.Alias,
|
||
IsPublicField: oldFd.IsPublicField,
|
||
FieldType: oldFd.FieldType,
|
||
}
|
||
if oldFd.Name == oldFd.Alias {
|
||
newFieldInfo.Alias = fd.Alias
|
||
}
|
||
newFieldsDescInfo = append(newFieldsDescInfo, newFieldInfo)
|
||
break
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// 走完上面这一步，newFieldsDescInfo就是事件列表具有的所有去重字段了
|
||
allEventFieldsLen := len(newFieldsDescInfo)
|
||
|
||
// 给newRows和newFieldsDescInfo前追加公共字段了
|
||
// 追加前面的日志名、时间
|
||
tmpFields := make([]*dto2.GameLogFieldInfo, 2)
|
||
copy(tmpFields, info.fields[:2])
|
||
newFieldsDescInfo = append(tmpFields, newFieldsDescInfo...)
|
||
for i, newRow := range newRows {
|
||
if len(newRow) < allEventFieldsLen {
|
||
rowCols := len(newRow)
|
||
for j := 0; j < allEventFieldsLen-rowCols; j++ {
|
||
// 补充这一行日志没有的字段
|
||
newRow = append(newRow, "\\")
|
||
}
|
||
} else if len(newRow) > allEventFieldsLen {
|
||
panic(fmt.Sprintf("%v,%v", len(newRow), allEventFieldsLen))
|
||
}
|
||
tmpCols := make([]any, 2)
|
||
copy(tmpCols, info.rows[i][:2])
|
||
newRow = append(tmpCols, newRow...)
|
||
newRows[i] = newRow
|
||
}
|
||
|
||
// 给newRows和newFieldsDescInfo后追加公共字段了
|
||
for i, f := range info.fields {
|
||
if len(f.Name) >= 4 && f.Name[:4] == "pub_" {
|
||
newFieldsDescInfo = append(newFieldsDescInfo, info.fields[i:]...)
|
||
for rowI, newRow := range newRows {
|
||
newRow = append(newRow, info.rows[rowI][i:]...)
|
||
newRows[rowI] = newRow
|
||
}
|
||
break
|
||
}
|
||
}
|
||
|
||
return newFieldsDescInfo, newRows
|
||
}
|
||
|
||
// 因为推送到clickhouse的字段名都是日志名_字段名的格式，放置不同日志出现同名字段冲突，这里直接去掉一下日志名
|
||
func removeFieldEventName(eventName, fieldName string) string {
|
||
return strings.ReplaceAll(fieldName, eventName+"_", "")
|
||
}
|