furryboard/spider-scheduler
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

实现通过直播间获取furry等

Browse Source 
用户信息新增直播间房间号字段

能用了，但不是很好用
This commit is contained in:
Eigeen
2022-12-25 18:06:03 +08:00
parent 511d889557
commit 4f44e60168
24 changed files with 2919 additions and 783 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
pkg/logic/config.toml
View File

@@ -10,3 +10,4 @@
[spider_core]
host = "localhost"
port = 9101
bili_cookie = "buvid3=E114B569-ECF7-50D6-5874-7221C956F4F323116infoc; i-wanna-go-back=-1; _uuid=9DC109D51-C164-D1A8-D18D-CDB410A6B733223200infoc; buvid4=2EFA9D48-F352-9314-066E-800EAD1A3B3024163-022061118-7XDfT9HnZ75xcCl6GNqbKvvAaotbbCqK1dff9AggS2AFysI2RsiDgg%3D%3D; CURRENT_BLACKGAP=0; blackside_state=0; nostalgia_conf=-1; buvid_fp_plain=undefined; b_ut=5; LIVE_BUVID=AUTO2216549597211800; hit-dyn-v2=1; b_nut=100; fingerprint3=8b45c9c8e2c3bde2334046651927bc11; CURRENT_QUALITY=80; hit-new-style-dyn=0; CURRENT_FNVAL=4048; rpdid=|(J~R|~)~|Y)0J'uYY)Y|JmkY; dy_spec_agreed=1; fingerprint=64f6f507fa149d279783afea30316e2b; SESSDATA=0312b6e2%2C1687167210%2C5c48e%2Ac1; bili_jct=12bdeb674ff87c20afd2d245128d6734; DedeUserID=90931399; DedeUserID__ckMd5=507c5d481d60b7bb; sid=5n2lvshx; buvid_fp=64f6f507fa149d279783afea30316e2b; bp_t_offset_90931399=742134893995098144; PVID=2; share_source_origin=QQ; bsource=share_source_qqchat; b_lsid=4578914A_1853786E484; innersign=0"

35
pkg/logic/liveroom.go Normal file
View File

@@ -0,0 +1,35 @@
package logic
import (
"github.com/eigeen/furryboard/spider-scheduler/pkg/dao"
"github.com/eigeen/furryboard/spider-scheduler/pkg/dao/model"
)
func GetLiveRoomsFromDB(roomIDs *[]uint64) ([]*model.LiveRoom, error) {
var liveRooms []*model.LiveRoom
tx := dao.DB().Model(&model.LiveRoom{}).Where("room_id in (?)", *roomIDs).Find(&liveRooms)
return liveRooms, tx.Error
}
//
//func UpdateLiveRoom(infos []*pb.InfoReply_Data) (int, error) {
// // 获取数据库里已有的info
// var roomIDs []uint64
// for _, info := range infos {
// if info.LiveRoom == nil {
// continue
// }
// roomIDs = append(roomIDs, info.LiveRoom.Roomid)
// }
// oldInfos, err := GetLiveRoomsFromDB(&roomIDs)
//
// err = dao.DB().Transaction(func(tx *gorm.DB) error {
// for i, info := range infos {
// if oldInfos[i].Title != info.LiveRoom
// }
// })
// if err != nil {
// return
// }
//
//}

42
pkg/logic/metadata.go Normal file
View File

@@ -0,0 +1,42 @@
package logic
import (
"errors"
"github.com/eigeen/furryboard/spider-scheduler/pkg/dao"
"github.com/eigeen/furryboard/spider-scheduler/pkg/dao/model"
"gorm.io/gorm"
)
func AddOrUpdateMetadata(key, value string) error {
metadata, err := GetMetadata(key)
if err != nil {
return err
}
var tx *gorm.DB
if metadata != nil {
metadata.Value = value
tx = dao.DB().Save(metadata)
} else {
tx = dao.DB().Create(&model.Metadata{
Key: key,
Value: value,
})
}
return tx.Error
}
func DeleteMetadata(key string) error {
tx := dao.DB().Model(&model.Metadata{}).Delete("key = ?", key)
return tx.Error
}
func GetMetadata(key string) (*model.Metadata, error) {
var metadata model.Metadata
tx := dao.DB().Model(&model.Metadata{}).Where("key = ?", key).First(&metadata)
err := tx.Error
if errors.Is(err, gorm.ErrRecordNotFound) {
return nil, nil
}
return &metadata, tx.Error
}

148
pkg/logic/search.go Normal file
View File

@@ -0,0 +1,148 @@
package logic
import (
"context"
"fmt"
"github.com/eigeen/furryboard/spider-scheduler/pkg/dao"
"github.com/eigeen/furryboard/spider-scheduler/pkg/dao/model"
"github.com/eigeen/furryboard/spider-scheduler/pkg/exception"
"github.com/eigeen/furryboard/spider-scheduler/rpc"
"github.com/eigeen/furryboard/spider-scheduler/rpc/pb"
"gorm.io/gorm"
"time"
)
// SearchNewestVideos b站搜索 最新发布排序 仅搜索视频
func SearchNewestVideos(keyword, cookie string, page uint) ([]*pb.SearchVideoResult, error) {
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
info, err := rpc.SpiderCore().SearchVideo(ctx, &pb.SearchVideoReq{
Keyword: keyword,
Order: "pubdate",
Page: uint32(page),
Cookie: cookie,
})
if err != nil {
return nil, exception.InternalError("获取最新视频搜索信息失败：" + err.Error())
}
if info.Code != 200 { // -400 -411请求被拦截
if info.Msg == "" {
info.Msg = "Unknown"
}
return nil, exception.ErrSearchVideo(
fmt.Sprintf("最新视频搜索信息[keyword=%s]获取失败：%s(Code: %d)", keyword, info.Msg, info.Code))
}
return info.Data.Result, nil
}
func UpdateVideoCheckpoint(value string) error {
return AddOrUpdateMetadata("SearchVideoCheckpoint", value)
}
// SearchLiveRooms b站搜索 默认排序 仅搜索直播间
func SearchLiveRooms(keyword, cookie string, page uint) ([]*pb.SearchLiveRoomResult, error) {
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
info, err := rpc.SpiderCore().SearchLiveRoom(ctx, &pb.SearchLiveRoomReq{
Keyword: keyword,
Order: "online",
Page: uint32(page),
Cookie: cookie,
})
if err != nil {
return nil, exception.InternalError("获取直播间搜索信息失败：" + err.Error())
}
if info.Code != 200 {
if info.Msg == "" {
info.Msg = "Unknown"
}
return nil, exception.ErrSearchLiveRoom(
fmt.Sprintf("直播间搜索信息[keyword=%s]获取失败：%s(Code: %d)", keyword, info.Msg, info.Code))
}
return info.Data.Result, nil
}
func GetLiveRoomPageNum(keyword, cookie string) (int, error) {
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
info, err := rpc.SpiderCore().SearchLiveRoom(ctx, &pb.SearchLiveRoomReq{
Keyword: keyword,
Order: "online",
Page: 1,
Cookie: cookie,
})
if err != nil {
return 0, exception.InternalError("获取直播间搜索信息失败：" + err.Error())
}
if info.Code != 200 {
if info.Msg == "" {
info.Msg = "Unknown"
}
return 0, exception.ErrSearchLiveRoom(
fmt.Sprintf("直播间搜索信息[keyword=%s]获取失败：%s(Code: %d)", keyword, info.Msg, info.Code))
}
return int(info.Data.NumPages), nil
}
// GetLiveRoomByRoomID 通过room_id从数据库查询直播间信息 返回值不为nil
func GetLiveRoomByRoomID(roomID uint64) (*model.LiveRoom, error) {
var room model.LiveRoom
tx := dao.DB().Model(&model.LiveRoom{}).Where("room_id = ?", roomID).Find(&room)
if tx.Error != nil {
return &room, tx.Error
}
return &room, nil
}
func UpdateLiveRoomInfo(rooms []*pb.SearchLiveRoomResult) (int, error) {
err := dao.DB().Transaction(func(tx *gorm.DB) error {
for _, room := range rooms {
roomInfo, err := GetLiveRoomByRoomID(room.Roomid)
if err != nil {
return err
}
roomInfo.RoomID = uint(room.Roomid)
roomInfo.Title = room.Title
roomInfo.Tags = room.Tags
if err = tx.Save(roomInfo).Error; err != nil {
return err
}
}
return nil
})
if err != nil {
return 0, err
}
err = dao.DB().Transaction(func(tx *gorm.DB) error {
for _, room := range rooms {
exists, err := IsUserExistsByUID(uint(room.Uid))
if err != nil {
return err
}
if !exists {
tx.Create(&model.Furry{
UID: uint(room.Uid),
Name: room.Uname,
Status: model.StatusPendingLive,
LiveRoomID: uint(room.Roomid),
})
continue
}
if err = tx.Model(&model.Furry{}).Where("uid = ?", room.Uid).Update("live_room_id", room.Roomid).Error; err != nil {
return err
}
}
return nil
})
if err != nil {
return 0, err
}
return len(rooms), nil
}

35
pkg/logic/search_test.go Normal file
View File

@@ -0,0 +1,35 @@
package logic
import (
"github.com/eigeen/furryboard/spider-scheduler/pkg/conf"
"testing"
)
func TestSearchNewestVideos(t *testing.T) {
BeforeTesting()
videos, err := SearchNewestVideos("furry", conf.Conf.SpiderCore.BiliCookie, 1)
if err != nil {
t.Error(err)
}
t.Log(len(videos))
t.Log(videos[0])
}
func TestGetLiveRoomPageNum(t *testing.T) {
BeforeTesting()
num, err := GetLiveRoomPageNum("furry", conf.Conf.SpiderCore.BiliCookie)
if err != nil {
t.Error(err)
}
t.Log(num)
}
func TestSearchLiveRooms(t *testing.T) {
BeforeTesting()
rooms, err := SearchLiveRooms("furry", conf.Conf.SpiderCore.BiliCookie, 1)
if err != nil {
t.Error(err)
}
t.Log(len(rooms))
t.Log(rooms[0])
}

38
pkg/logic/user.go
View File

@@ -16,11 +16,11 @@ import (
"time"
)
// GetUsers 分页获取用户信息（数据库）
func GetUsers(page, pageSize int) ([]*model.Furry, error) {
// GetValidUsers 分页获取用户信息（数据库）
func GetValidUsers(page, pageSize int) ([]*model.Furry, error) {
// 分页查询
var furries []*model.Furry
tx := dao.DB().Offset((page - 1) * pageSize).Limit(pageSize).Find(&furries)
tx := dao.DB().Where("status = 0").Offset((page - 1) * pageSize).Limit(pageSize).Find(&furries)
if tx.Error != nil {
return nil, exception.ErrFetchFurries("分页获取Furry列表时失败：" + tx.Error.Error())
}
@@ -45,6 +45,20 @@ func GetUserInfoByUID(uid uint) (*pb.InfoReply_Data, error) {
return info.Data, nil
}
// IsUserExistsByUID 通过UID判断用户是否存在数据库
func IsUserExistsByUID(uid uint) (bool, error) {
var count int64
tx := dao.DB().Model(&model.Furry{}).Where("uid = ?", uid).Count(&count)
if tx.Error != nil {
return false, exception.ErrFetchFurries(fmt.Sprintf("查找用户[%d]时发生错误：%s", uid, tx.Error.Error()))
}
if count == 0 {
return false, nil
} else {
return true, nil
}
}
// BatchGetUserInfo 批量获取用户基础信息
func BatchGetUserInfo(userIds *[]uint) []*pb.InfoReply_Data {
poolSize := 8
@@ -82,8 +96,8 @@ func BatchGetUserInfo(userIds *[]uint) []*pb.InfoReply_Data {
func UpdateUserInfo(users []*model.Furry, infos []*pb.InfoReply_Data) (int, error) {
// 以UID为索引转换为map
infoMap := make(map[uint]*pb.InfoReply_Data)
for _, i := range infos {
infoMap[uint(i.Mid)] = i
for _, info := range infos {
infoMap[uint(info.Mid)] = info
}
// 检测变化
changelogs, changes, err := createChangelogs(users, infos)
@@ -120,7 +134,7 @@ func userInfoDiff(user *model.Furry, info *pb.InfoReply_Data) (*model.Changelog,
return nil, nil, exception.ErrUserMismatch(fmt.Sprintf("对比用户ID不匹配：%d和%d", user.UID, info.Mid))
}
changelog := model.Changelog{
Operation: "UpdateName",
Operation: "",
Target: strconv.FormatUint(uint64(user.UID), 10),
Result: "",
}
@@ -129,9 +143,15 @@ func userInfoDiff(user *model.Furry, info *pb.InfoReply_Data) (*model.Changelog,
}
if user.Name != info.Name {
result, _ := json.Marshal(map[string]string{"previous_name": user.Name, "new_name": info.Name})
changelog.Operation = "UpdateName"
changelog.Result = string(result)
updated.Name = info.Name
return &changelog, &updated, nil
} else if info.LiveRoom != nil && user.LiveRoomID != uint(info.LiveRoom.Roomid) {
updated.LiveRoomID = uint(info.LiveRoom.Roomid)
changelog.Operation = "AddLiveRoom"
changelog.Result = strconv.FormatUint(info.LiveRoom.Roomid, 10)
return &changelog, &updated, nil
} else {
return nil, nil, nil
}
@@ -238,5 +258,9 @@ func UpdateFansToDB(stats []*pb.StatReply_Data) (int, error) {
return 0, err
}
return len(stats), nil
}
func AddUser(user *model.Furry) error {
tx := dao.DB().Create(user)
return tx.Error
}