uniugm/admin/apps/game/domain/projects/smdl/gamelog.go

package smdl

import (
	"admin/apps/game/domain/entity"
	dto2 "admin/internal/model/dto"
	"admin/lib/xlog"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"strconv"
	"strings"
	"time"
)

type GameLogHook struct {
}

var removeFields = map[string]struct{}{
	"pub_mediaid":   {},
	"pub_role_name": {},
	"pub_udid":      {},
}

var fieldsAlias = map[string]string{
	"pub_viplev":    "vip等级",
	"pub_userid":    "账号",
	"pub_totalcash": "总充值金额",
	"pub_serverid":  "服务器id",
	"pub_rolename":  "角色名",
	"pub_roleid":    "角色id",
	"pub_lev":       "等级",
	"pub_language":  "语言",
	"pub_ip":        "IP",
}

var base64decodeFields = map[string]struct{}{
	"pub_rolename": {},
	"chatlog_msg":  {},
}

var reasonValueAlias = map[string]map[string]string{
	"gainitem_itempath":  reasonGainDesc,
	"loseitem_itempath":  reasonCostDesc,
	"addcoin_reasonstr":  reasonGainDesc,
	"costcoin_reasonstr": reasonCostDesc,
}

var reasonEnumValueAlias = map[string]map[int]string{
	"addcoin_coinpath":  reasonCostEnumDesc,
	"costcoin_coinpath": reasonCostEnumDesc,
}

func init() {
	for i := 0; i < 15; i++ {
		removeFields["chatlog_score"] = struct{}{}
		removeFields["chatlog_score"+strconv.Itoa(i)] = struct{}{}
		removeFields["chatlog_label"] = struct{}{}
		removeFields["chatlog_label"+strconv.Itoa(i)] = struct{}{}
		removeFields["chatlog_checklevel"] = struct{}{}
		removeFields["chatlog_checklevel"+strconv.Itoa(i)] = struct{}{}
	}
	removeFields["chatlog_code"] = struct{}{}
	removeFields["chatlog_checkresult"] = struct{}{}
}

func (hook *GameLogHook) Trim(projectInfo *entity.Project, eventName []string, totalCount int, fieldsDescInfo []*dto2.GameLogFieldInfo, rows [][]any) (
	int, []*dto2.GameLogFieldInfo, [][]any) {
	totalCount, fieldsDescInfo, rows = hook.getTrimHandlerByEventName(eventName)(projectInfo, totalCount, fieldsDescInfo, rows)
	return totalCount, fieldsDescInfo, rows
}

func (hook *GameLogHook) getTrimHandlerByEventName(eventName []string) func(projectInfo *entity.Project, totalCount int, fieldsDescInfo []*dto2.GameLogFieldInfo, rows [][]any) (
	int, []*dto2.GameLogFieldInfo, [][]any) {
	for _, v := range eventName {
		if v == "chatlog" {
			return hook.trimChat
		}
		if v == "addcoin" || v == "costcoin" {
			return hook.trimCurrencyChange
		}
		if v == "gainitem" || v == "loseitem" {
			return hook.trimItemChange
		}
	}
	return nil
}

func (hook *GameLogHook) trimChat(projectInfo *entity.Project, totalCount int, fieldsDescInfo []*dto2.GameLogFieldInfo, rows [][]any) (
	int, []*dto2.GameLogFieldInfo, [][]any) {
	newFields, newRows := hook.trimCommon(projectInfo, fieldsDescInfo, rows)

	extractFields := [][2]string{{"chatlog_channel", "渠道"}, {"chatlog_msg", "聊天内容"}}

	for _, eField := range extractFields {
		for colI, field := range fieldsDescInfo {
			if field.Name == eField[0] {
				field.Alias = eField[1]
				newFields = append(newFields, field)
				for rowI, row := range rows {
					if field.Name == "chatlog_msg" {
						newValue, err := base64.StdEncoding.DecodeString(row[colI].(string))
						if err != nil {
							xlog.Warnf("base64 decode field:%v value:%v error:%v", field.Name, row[colI], err)
						} else {
							newRows[rowI] = append(newRows[rowI], string(newValue))
						}
					} else if field.Name == "chatlog_channel" {
						channel := int(row[colI].(float64))
						channelDesc, find := chatChannels[channel]
						if find {
							newRows[rowI] = append(newRows[rowI], channelDesc)
						} else {
							newRows[rowI] = append(newRows[rowI], fmt.Sprintf("%v", row[colI]))
						}
					}
				}
				break
			}
		}
	}

	return totalCount, newFields, newRows
}

func (hook *GameLogHook) trimCurrencyChange(projectInfo *entity.Project, totalCount int, fieldsDescInfo []*dto2.GameLogFieldInfo, rows [][]any) (
	int, []*dto2.GameLogFieldInfo, [][]any) {
	newFields, newRows := hook.trimCommon(projectInfo, fieldsDescInfo, rows)
	newFields = append(newFields, []*dto2.GameLogFieldInfo{
		{Name: "coinType", Alias: "货币类型"},
		{Name: "coinPath", Alias: "途径"},
		{Name: "beforeNum", Alias: "改变前数量"},
		{Name: "deltaNum", Alias: "改变数量"},
		{Name: "afterNum", Alias: "改变后数量"},
	}...)
	extractFields := map[string][]string{
		"addcoin":  {"addcoin_cointype", "addcoin_coinpath", "addcoin_coin", "addcoin_coinleft"},
		"costcoin": {"costcoin_cointype", "costcoin_coinpath", "costcoin_coin", "costcoin_coinleft"},
	}
	// 刷新一下缓存
	new(Items).GetItems(projectInfo)
	for rowI, row := range rows {
		eventName := row[0].(string)
		for _, eFieldName := range extractFields[eventName] {
			for fieldI, field := range fieldsDescInfo {
				if field.Name == eFieldName {
					colValue := row[fieldI]
					if field.Name == "addcoin_cointype" || field.Name == "costcoin_cointype" {
						// 货币类型要获取对应中文描述
						coinType := int(colValue.(float64))
						coinDesc := currencyDescTypes[coinType]
						colValue = coinDesc
					} else if field.Name == "addcoin_coinpath" {
						desc, find := reasonGainEnumDesc[int(colValue.(float64))]
						if find {
							colValue = desc
						}
					} else if field.Name == "costcoin_coinpath" {
						desc, find := reasonCostEnumDesc[int(colValue.(float64))]
						if find {
							colValue = desc
						}
					}
					newRows[rowI] = append(newRows[rowI], colValue)
					break
				}
			}
		}
	}

	// 计算一下改变前数量
	for i, row := range newRows {
		deltaNum := row[len(row)-2].(float64)
		leftCount := row[len(row)-1].(float64)
		if row[0].(string) == "costcoin" {
			deltaNum = -deltaNum
		}

		beforeNum := leftCount - deltaNum
		row = row[:len(row)-2]
		row = append(row, beforeNum)
		row = append(row, deltaNum)
		row = append(row, leftCount)
		newRows[i] = row
	}

	return totalCount, newFields, newRows
}

func (hook *GameLogHook) trimItemChange(projectInfo *entity.Project, totalCount int, fieldsDescInfo []*dto2.GameLogFieldInfo, rows [][]any) (
	int, []*dto2.GameLogFieldInfo, [][]any) {
	newFields, newRows := hook.trimCommon(projectInfo, fieldsDescInfo, rows)
	newFields = append(newFields, []*dto2.GameLogFieldInfo{
		{Name: "itemid", Alias: "道具id"},
		{Name: "itemName", Alias: "道具名称"},
		{Name: "reason", Alias: "途径"},
		{Name: "beforeNum", Alias: "改变前数量"},
		{Name: "deltaNum", Alias: "改变数量"},
		{Name: "afterNum", Alias: "改变后数量"},
	}...)
	extractFields := map[string][]string{
		"gainitem": {"gainitem_itemid", "gainitem_itempath", "gainitem_itemcount", "gainitem_itemleft"},
		"loseitem": {"loseitem_itemid", "loseitem_itempath", "loseitem_itemcount", "loseitem_itemleft"},
	}
	// 刷新一下缓存
	new(Items).GetItems(projectInfo)
	for rowI, row := range rows {
		eventName := row[0].(string)
		for _, eFieldName := range extractFields[eventName] {
			for fieldI, field := range fieldsDescInfo {
				if field.Name == eFieldName {
					colValue := row[fieldI]
					if field.Name == "gainitem_itemid" || field.Name == "loseitem_itemid" {
						// 道具id要获取对应道具名
						itemId := int(colValue.(float64))
						if itemsCache != nil {
							itemInfo, find := itemsCache.mapItems[itemId]
							if find {
								itemName := itemInfo.Desc
								newRows[rowI] = append(newRows[rowI], itemId)
								newRows[rowI] = append(newRows[rowI], itemName)
							} else {
								newRows[rowI] = append(newRows[rowI], itemId)
								newRows[rowI] = append(newRows[rowI], strconv.Itoa(itemId))
							}
						} else {
							newRows[rowI] = append(newRows[rowI], itemId)
							newRows[rowI] = append(newRows[rowI], strconv.Itoa(itemId))
						}
					} else if field.Name == "gainitem_itempath" {
						desc, find := reasonGainDesc[colValue.(string)]
						if find {
							newRows[rowI] = append(newRows[rowI], desc)
						} else {
							newRows[rowI] = append(newRows[rowI], colValue)
						}
					} else if field.Name == "loseitem_itempath" {
						desc, find := reasonCostDesc[colValue.(string)]
						if find {
							newRows[rowI] = append(newRows[rowI], desc)
						} else {
							newRows[rowI] = append(newRows[rowI], colValue)
						}
					} else {
						newRows[rowI] = append(newRows[rowI], colValue)
					}
					break
				}
			}
		}
	}

	// 计算一下改变前数量
	for i, row := range newRows {
		deltaNum := row[len(row)-2].(float64)
		leftCount := row[len(row)-1].(float64)
		if row[0].(string) == "loseitem" {
			deltaNum = -deltaNum
		}

		beforeNum := leftCount - deltaNum
		row = row[:len(row)-2]
		row = append(row, beforeNum)
		row = append(row, deltaNum)
		row = append(row, leftCount)
		newRows[i] = row
	}

	return totalCount, newFields, newRows
}

var commonExtractFields = [][2]string{
	{"xwl_part_event", "事件类型"},
	{"xwl_part_date", "时间"},
	{"pub_userid", "账号"},
	{"pub_roleid", "角色id"},
	{"pub_rolename", "角色名"},
}

func (hook *GameLogHook) trimCommon(projectInfo *entity.Project, fieldsDescInfo []*dto2.GameLogFieldInfo, rows [][]any) (
	[]*dto2.GameLogFieldInfo, [][]any) {
	newFields := make([]*dto2.GameLogFieldInfo, 0, 10)
	newRows := make([][]any, len(rows))
	for _, eField := range commonExtractFields {
		for i, field := range fieldsDescInfo {
			if eField[0] == field.Name {
				field.Alias = eField[1]
				newFields = append(newFields, field)
				for j, newRow := range newRows {
					if field.Name == "pub_rolename" {
						newValue, err := base64.StdEncoding.DecodeString(rows[j][i].(string))
						if err != nil {
							xlog.Warnf("base64 decode field:%v value:%v error:%v", field.Name, rows[j][i], err)
						} else {
							newRows[j] = append(newRows[j], string(newValue))
						}
					} else if field.Name == "xwl_part_date" {
						v := rows[j][i].(time.Time)
						newRows[j] = append(newRow, v.Format(time.DateTime))
					} else {
						newRows[j] = append(newRow, rows[j][i])
					}
				}
				break
			}
		}
	}
	return newFields, newRows
}

func (hook *GameLogHook) TrimBak(projectInfo *entity.Project, eventName []string, totalCount int, fieldsDescInfo []*dto2.GameLogFieldInfo, rows [][]any) (
	int, []*dto2.GameLogFieldInfo, [][]any) {

	// 删除不需要的字段
	i := 0
	for {
		if i >= len(fieldsDescInfo) {
			break
		}
		field := fieldsDescInfo[i]
		if _, find := removeFields[field.Name]; find {
			// 找到这个字段在去掉的列表里
			fieldsDescInfo = append(fieldsDescInfo[:i], fieldsDescInfo[i+1:]...)
			// 对应所有数据行里也删除这个值
			for rowI, row := range rows {
				row = append(row[:i], row[i+1:]...)
				rows[rowI] = row
			}
		} else {
			// 查找要不要把value做base64解码
			if _, find := base64decodeFields[field.Name]; find {
				for _, row := range rows {
					newValue, err := base64.StdEncoding.DecodeString(row[i].(string))
					if err != nil {
						xlog.Warnf("base64 decode field:%v value:%v error:%v", field.Name, row[i], err)
					} else {
						row[i] = string(newValue)
					}
				}
			}
			i++
		}
	}

	//fBin, _ := json.Marshal(&fieldsDescInfo)
	//rBin, _ := json.Marshal(&rows)
	//xlog.Tracef("gamelog1 query result:%v, rows:%v", string(fBin), string(rBin))

	for i, f := range fieldsDescInfo {
		// 修改每一行日志值描述
		if replaceMap, findReplaceValue := reasonValueAlias[f.Name]; findReplaceValue {
			for j := range rows {
				curValue, ok := rows[j][i].(string)
				if ok {
					newValue, find := replaceMap[curValue]
					if find {
						rows[j][i] = newValue
					}
				}
			}
		} else if replaceMap, findReplaceValue := reasonEnumValueAlias[f.Name]; findReplaceValue {
			for j := range rows {
				curValue, ok := rows[j][i].(float64)
				if ok {
					newValue, find := replaceMap[int(curValue)]
					if find {
						rows[j][i] = newValue
					}
				}
			}
		}
	}

	fieldsDescInfo, rows = (&queryResultInfo{fields: fieldsDescInfo, rows: rows}).tidyByEventDescInfo(eventName)

	//fBin, _ = json.Marshal(&fieldsDescInfo)
	//rBin, _ = json.Marshal(&rows)
	//xlog.Tracef("gamelog1 query result:%v, rows:%v", string(fBin), string(rBin))

	for i, f := range fieldsDescInfo {
		// 修改每一行日志别名
		if f.Name == "xwl_part_event" {
			for j := range rows {
				opEventName := rows[j][i].(string)
				descInfo, find := globEventList[opEventName]
				if find {
					rows[j][i] = descInfo.Alias
				}
			}
		} else if f.Name == "xwl_part_date" {
			for j := range rows {
				eventDate := rows[j][i].(time.Time)
				//eventDate, err := time.Parse("2006-01-02T15:04:05+07:00", eventDateString)
				rows[j][i] = eventDate.Format(time.DateTime)
			}
		}

		// 修改每一行公共属性别名
		alias, find := fieldsAlias[f.Name]
		if find && f.Name == f.Alias {
			f.Alias = alias
		}
	}

	fBin, _ := json.Marshal(&fieldsDescInfo)
	rBin, _ := json.Marshal(&rows)
	xlog.Tracef("gamelog2 query result:%v, rows:%v", string(fBin), string(rBin))

	i = 0
	for {
		if i >= len(fieldsDescInfo) {
			break
		}
		f := fieldsDescInfo[i]
		swapI := 2
		if f.Name == "pub_serverid" || f.Name == "pub_roleid" || f.Name == "pub_rolename" {
			pubF := fieldsDescInfo[i]
			for moveI := i; moveI > swapI; moveI-- {
				fieldsDescInfo[moveI] = fieldsDescInfo[moveI-1]
			}
			fieldsDescInfo[swapI] = pubF

			for _, row := range rows {
				pubValue := row[i]
				for moveI := i; moveI > swapI; moveI-- {
					row[moveI] = row[moveI-1]
				}
				row[swapI] = pubValue
			}
			swapI++
		}
		i++
	}

	return totalCount, fieldsDescInfo, rows
}

type queryResultInfo struct {
	fields []*dto2.GameLogFieldInfo
	rows   [][]any
}

func (info *queryResultInfo) tidyByEventDescInfo(eventNameList []string) ([]*dto2.GameLogFieldInfo, [][]any) {
	if len(eventNameList) == 0 {
		return info.fields, info.rows
	}
	if len(eventNameList) == 1 {
		en := eventNameList[0]
		enDescInfo, find := globEventList[en]
		if !find {
			panic(en)
		}
		for _, f := range info.fields {
			for _, df := range enDescInfo.fields {
				fName := removeFieldEventName(en, f.Name)
				if fName == df.Name && f.Name == f.Alias {
					f.Alias = df.Alias
					break
				}
			}
		}
		return info.fields, info.rows
	}

	// 神魔的埋点里字段名是日志名_字段名，所以如果一次游戏日志查询包含多个事件名，会查出稀疏数据，
	// 例如gainitem,loseitem两个日志，都会存在gainitem_itemid,loseitem_itemid，需要把对应
	// 为0的稀疏字段去掉

	// 具体方案：
	// 1.创建新的行数据，将自己日志的字段值先追加进来，然后前面补上日志名，后面补上公共字段，
	// 2.字段描述信息直接全部去重

	newRows := make([][]any, len(info.rows))
	for rowI, row := range info.rows {
		curRowEventName := row[0].(string)
		enDescInfo, find := globEventList[curRowEventName]
		if !find {
			// 不存在描述信息，先崩溃下
			panic(curRowEventName)
		}

		for _, fieldDesc := range enDescInfo.fields {
			for i, field := range info.fields {
				fName := removeFieldEventName(curRowEventName, field.Name)
				if fName == fieldDesc.Name {
					newRows[rowI] = append(newRows[rowI], row[i])
					break
				}
			}
		}
	}

	// 上面步骤走完，newRows每一行都是各自日志的字段值了，但是每一行长度可能不一样，例如gainitem就比loseitem多一个字段
	// 下面步骤会把字段描述信息通过事件名具有的所有字段去重追加，例如gainitem就是全量的字段，loseitem就是子集
	newFieldsDescInfo := make([]*dto2.GameLogFieldInfo, 0, len(info.fields))
	for _, en := range eventNameList {
		enDescInfo, find := globEventList[en]
		if !find {
			panic(en)
		}
		find = false
		for _, fd := range enDescInfo.fields {
			for _, newFd := range newFieldsDescInfo {
				if fd.Name == newFd.Name {
					find = true
					break
				}
			}
			if !find {
				// 追加
				for _, oldFd := range info.fields {
					fName := removeFieldEventName(en, oldFd.Name)
					if fName == fd.Name {
						newFieldInfo := &dto2.GameLogFieldInfo{
							Name:          fName,
							Alias:         oldFd.Alias,
							IsPublicField: oldFd.IsPublicField,
							FieldType:     oldFd.FieldType,
						}
						if oldFd.Name == oldFd.Alias {
							newFieldInfo.Alias = fd.Alias
						}
						newFieldsDescInfo = append(newFieldsDescInfo, newFieldInfo)
						break
					}
				}
			}
		}
	}

	// 走完上面这一步，newFieldsDescInfo就是事件列表具有的所有去重字段了
	allEventFieldsLen := len(newFieldsDescInfo)

	// 给newRows和newFieldsDescInfo前追加公共字段了
	// 追加前面的日志名、时间
	tmpFields := make([]*dto2.GameLogFieldInfo, 2)
	copy(tmpFields, info.fields[:2])
	newFieldsDescInfo = append(tmpFields, newFieldsDescInfo...)
	for i, newRow := range newRows {
		if len(newRow) < allEventFieldsLen {
			rowCols := len(newRow)
			for j := 0; j < allEventFieldsLen-rowCols; j++ {
				// 补充这一行日志没有的字段
				newRow = append(newRow, "\\")
			}
		} else if len(newRow) > allEventFieldsLen {
			panic(fmt.Sprintf("%v,%v", len(newRow), allEventFieldsLen))
		}
		tmpCols := make([]any, 2)
		copy(tmpCols, info.rows[i][:2])
		newRow = append(tmpCols, newRow...)
		newRows[i] = newRow
	}

	// 给newRows和newFieldsDescInfo后追加公共字段了
	for i, f := range info.fields {
		if len(f.Name) >= 4 && f.Name[:4] == "pub_" {
			newFieldsDescInfo = append(newFieldsDescInfo, info.fields[i:]...)
			for rowI, newRow := range newRows {
				newRow = append(newRow, info.rows[rowI][i:]...)
				newRows[rowI] = newRow
			}
			break
		}
	}

	return newFieldsDescInfo, newRows
}

// 因为推送到clickhouse的字段名都是日志名_字段名的格式，放置不同日志出现同名字段冲突，这里直接去掉一下日志名
func removeFieldEventName(eventName, fieldName string) string {
	return strings.ReplaceAll(fieldName, eventName+"_", "")
}