fix: empty requests

const.go

@@ -9,6 +9,7 @@ const (
 	// Base URL
 	tiktokBaseUrl = "https://www.tiktok.com/"
 	tiktokAPIUrl  = "https://webcast.tiktok.com/webcast/"
+	tiktokSigner  = "https://tiktok.isaackogan.com/"
 
 	// Endpoints
 	urlLive      = "live/"
@@ -20,38 +21,49 @@ const (
 	urlRoomInfo = "room/info/"
 	urlRoomData = "im/fetch/"
 	urlGiftInfo = "gift/list/"
+	urlSignReq  = "webcast/sign_url/"
 
 	// Default Request Headers
-	userAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.99 Safari/537.36"
+	userAgent = "5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.5005.63 Safari/537.36"
 	referer   = "https://www.tiktok.com/"
 	origin    = "https://www.tiktok.com"
+	clientId  = "ttlive-golang"
 )
 
 var (
 	// Default GET Request parameters
 	defaultGETParams = map[string]string{
-		"aid":               "1988",
-		"app_language":      "en-US",
-		"app_name":          "tiktok_web",
-		"browser_language":  "en",
-		"browser_name":      "Mozilla",
-		"browser_online":    "true",
-		"browser_platform":  "Win32",
-		"browser_version":   "5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.99 Safari/537.36",
-		"cookie_enabled":    "true",
-		"cursor":            "",
-		"device_platform":   "web",
-		"did_rule":          "3",
-		"fetch_rule":        "1",
-		"identity":          "audience",
-		"internal_ext":      "",
-		"last_rtt":          "0",
-		"live_id":           "12",
-		"resp_content_type": "protobuf",
-		"screen_height":     "1152",
-		"screen_width":      "2048",
-		"tz_name":           "Europe/Berlin",
-		"version_code":      "180800",
+		"aid":                 "1988",
+		"app_language":        "en-US",
+		"app_name":            "tiktok_web",
+		"browser_language":    "en",
+		"browser_name":        "Mozilla",
+		"browser_online":      "true",
+		"browser_platform":    "Win32",
+		"browser_version":     "5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.5005.63 Safari/537.36",
+		"cookie_enabled":      "true",
+		"cursor":              "",
+		"internal_ext":        "",
+		"device_platform":     "web",
+		"focus_state":         "true",
+		"from_page":           "user",
+		"history_len":         "4",
+		"is_fullscreen":       "false",
+		"is_page_visible":     "true",
+		"did_rule":            "3",
+		"fetch_rule":          "1",
+		"identity":            "audience",
+		"last_rtt":            "0",
+		"live_id":             "12",
+		"resp_content_type":   "protobuf",
+		"screen_height":       "1152",
+		"screen_width":        "2048",
+		"tz_name":             "Europe/Berlin",
+		"referer":             "https://www.tiktok.com/",
+		"root_referer":        "https://www.tiktok.com/",
+		"version_code":        "180800",
+		"webcast_sdk_version": "1.3.0",
+		"update_version_code": "1.3.0",
 	}
 	reJsonData = []*regexp.Regexp{
 		regexp.MustCompile(`<script id="SIGI_STATE"[^>]+>(.*?)</script>`),
@@ -69,5 +81,6 @@ var (
 	ErrUserNotFound      = errors.New("User not found")
 	ErrCaptcha           = errors.New("Captcha detected, unable to proceed")
 	ErrUrlNotFound       = errors.New("Unable to download stream, URL not found.")
-	ErrFFMPEGNotFound = errors.New("Please install ffmpeg before downloading.")
+	ErrFFMPEGNotFound    = errors.New("Please install ffmpeg before downloading.")
+	ErrRateLimitExceeded = errors.New("You have exceeded the rate limit, please wait a few min")
 )

feed_test.go

@@ -1,6 +1,11 @@
 package gotiktoklive
 
-import "testing"
+import (
+	"sort"
+	"testing"
+
+	"github.com/Davincible/gotiktoklive/tests"
+)
 
 func TestFeedItem(t *testing.T) {
 	tiktok := NewTikTok()
@@ -24,6 +29,12 @@ func TestFeedItem(t *testing.T) {
 			break
 		}
 	}
-
 	t.Logf("Found %d items, over %d requests", len(items), i)
+
+	sort.Slice(items, func(i, j int) bool {
+		return items[i].Room.UserCount > items[j].Room.UserCount
+	})
+
+	tests.USERNAME = items[0].Room.Owner.Username
+	t.Logf("Setting username to %s", items[0].Room.Owner.Username)
 }

go.mod

@@ -5,12 +5,20 @@ go 1.17
 require github.com/gobwas/ws v1.1.0
 
 require (
+	github.com/chromedp/sysutil v1.0.0 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
+)
+
+require (
+	github.com/chromedp/cdproto v0.0.0-20220812200530-d0d83820bffc
+	github.com/chromedp/chromedp v0.8.3
 	github.com/gobwas/httphead v0.1.0 // indirect
 	github.com/gobwas/pool v0.2.1 // indirect
 	github.com/pkg/errors v0.9.1
-	golang.org/x/net v0.0.0-20220225172249-27dd8689420f
-	golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e // indirect
-	google.golang.org/protobuf v1.27.1
+	golang.org/x/net v0.0.0-20220812174116-3211cb980234
+	golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab // indirect
+	google.golang.org/protobuf v1.28.1
 )
 
 replace github.com/Davincible/gotiktoklive => ./

go.sum

@@ -1,3 +1,10 @@
+github.com/chromedp/cdproto v0.0.0-20220725225757-5988d9195a6c/go.mod h1:5Y4sD/eXpwrChIuxhSr/G20n9CdbCmoerOHnuAf0Zr0=
+github.com/chromedp/cdproto v0.0.0-20220812200530-d0d83820bffc h1:xGhCpiX5oNMoZGnzYvv1ne4muVRl2SDHH5fL7oUbZAY=
+github.com/chromedp/cdproto v0.0.0-20220812200530-d0d83820bffc/go.mod h1:5Y4sD/eXpwrChIuxhSr/G20n9CdbCmoerOHnuAf0Zr0=
+github.com/chromedp/chromedp v0.8.3 h1:UwOY+fhC5Vv3uKgRpnvilCbWs/QPz8ciFwRB0q6pH8k=
+github.com/chromedp/chromedp v0.8.3/go.mod h1:9YfKSJnBNeP77vKecv+DNx2/Tcb+6Gli0d1aZPw/xbk=
+github.com/chromedp/sysutil v1.0.0 h1:+ZxhTpfpZlmchB58ih/LBHX52ky7w2VhQVKQMucy3Ic=
+github.com/chromedp/sysutil v1.0.0/go.mod h1:kgWmDdq8fTzXYcKIBqIYvRRTnYb9aNS9moAV0xufSww=
 github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU=
 github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM=
 github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og=
@@ -7,19 +14,27 @@ github.com/gobwas/ws v1.1.0/go.mod h1:nzvNcVha5eUziGrbxFCo6qFIojQHjJV5cLYIbezhfL
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
 github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
+github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
+github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/orisano/pixelmatch v0.0.0-20210112091706-4fa4c7ba91d5 h1:1SoBaSPudixRecmlHXb/GxmaD3fLMtHIDN13QujwQuc=
+github.com/orisano/pixelmatch v0.0.0-20210112091706-4fa4c7ba91d5/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-golang.org/x/net v0.0.0-20220225172249-27dd8689420f h1:oA4XRj0qtSt8Yo1Zms0CUlsT3KG69V2UGQWPBxujDmc=
-golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.0.0-20220812174116-3211cb980234 h1:RDqmgfe7SvlMWoqC3xwQ2blLO3fcWcxMa3eBLRdRW7E=
+golang.org/x/net v0.0.0-20220812174116-3211cb980234/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
 golang.org/x/sys v0.0.0-20201207223542-d4d67f95c62d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM=
-golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab h1:2QkjZIsXupsJbJIdSjjUOgWK3aEtzyuh2mPt3l/CkeU=
+golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
-google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
-google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w=
+google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=

live.go

@@ -7,6 +7,7 @@ import (
 	"net"
 	"os"
 	"os/exec"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -51,6 +52,9 @@ func (t *TikTok) newLive(roomId string) *Live {
 		Events:   make(chan interface{}, DEFAULT_EVENTS_CHAN_SIZE),
 		chanSize: DEFAULT_EVENTS_CHAN_SIZE,
 	}
+	t.mu.Lock()
+	t.streams += 1
+	t.mu.Unlock()
 
 	ctx, cancel := context.WithCancel(context.Background())
 	live.done = ctx.Done
@@ -60,6 +64,9 @@ func (t *TikTok) newLive(roomId string) *Live {
 			cancel()
 			t.wg.Wait()
 			close(live.Events)
+			t.mu.Lock()
+			t.streams -= 1
+			t.mu.Unlock()
 		})
 	}
 
@@ -121,6 +128,11 @@ func (t *TikTok) TrackUser(username string) (*Live, error) {
 		return nil, err
 	}
 
+	t.sendRequest(&reqOptions{
+		Endpoint: fmt.Sprintf(urlUser, username) + "live",
+		OmitAPI:  true,
+	})
+
 	return t.TrackRoom(id)
 }
 
@@ -318,7 +330,11 @@ func (l *Live) DownloadStream(file ...string) error {
 	}
 
 	// Run ffmpeg command
-	cmd := exec.Command("ffmpeg", "-i", url, "-c", "copy", path)
+	c := []string{"-i", url, "-c", "copy", path}
+	if l.t.proxy != nil && (l.t.proxy.Scheme == "http" || l.t.proxy.Scheme == "https") {
+		c = append([]string{"-http_proxy", l.t.proxy.String()}, c...)
+	}
+	cmd := exec.Command("ffmpeg", c...)
 
 	stdin, err := cmd.StdinPipe()
 	if err != nil {
@@ -364,10 +380,9 @@ func (l *Live) DownloadStream(file ...string) error {
 		if err := cmd.Wait(); err != nil {
 			nerr := new(exec.ExitError)
 			if errors.As(err, &nerr) {
-				l.t.errHandler(fmt.Errorf("Download command failed with: %w\nCommand: %v\nStderr: %v\nStdout: %v\n", err, cmd.Args, string(stderrb), string(stdoutb)))
-
+				l.t.errHandler(fmt.Errorf("download command failed with: %w\nCommand: %v\nStderr: %v\nStdout: %v\n", err, cmd.Args, string(stderrb), string(stdoutb)))
 			}
-			l.t.errHandler(fmt.Errorf("Download command failed with: %w\nCommand: %v\n", err, cmd.Args))
+			l.t.errHandler(fmt.Errorf("download command failed with: %w\nCommand: %v\n", err, cmd.Args))
 		}
 
 		mu.Lock()
@@ -381,6 +396,28 @@ func (l *Live) DownloadStream(file ...string) error {
 	return nil
 }
 
+func (t *TikTok) signURL(reqUrl string) (*SignedURL, error) {
+	body, err := t.sendRequest(&reqOptions{
+		URI:      tiktokSigner,
+		Endpoint: urlSignReq,
+		Query: map[string]string{
+			"client":            clientId,
+			"uuc":               strconv.Itoa(t.streams),
+			"url":               reqUrl,
+			"extendedRateLimit": "yes",
+		},
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "Failed to sign request")
+	}
+
+	var data SignedURL
+	if err := json.Unmarshal(body, &data); err != nil {
+		return nil, errors.Wrap(err, "Failed to unmarshal signer server json response")
+	}
+	return &data, nil
+}
+
 // Only able to get this while logged in
 // func (l *Live) GetRankList() (*RankList, error) {
 // 	t := l.t

live_test.go

@@ -76,3 +76,15 @@ func TestRoomData(t *testing.T) {
 
 	t.Logf("Ws url: %s, %+v", live.wsURL, live.wsParams)
 }
+
+// func TestHeadless(t *testing.T) {
+// 	tiktok := NewTikTok()
+// 	tiktok.SetProxy("http://127.0.0.1:8080", false)
+// 	// err := tiktok.openTikTok("https://intoli.com/blog/not-possible-to-block-chrome-headless/chrome-headless-test.html")
+// 	err := tiktok.openTikTok("https://www.tiktok.com/")
+// 	if err != nil {
+// 		t.Fatal(err)
+// 	}
+//
+// 	t.Log("Test done!")
+// }