Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[pull] main from twitter:main #3

Merged
merged 3 commits into from
Apr 5, 2023
Merged

[pull] main from twitter:main #3

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
3f69746
[VF] updates includes addressing Ukraine labels
Apr 4, 2023
3496189
[docs] Fix broken navi link in README
Apr 4, 2023
138bb51
[cr-mixer/home-mixer] Remove `getLinearRankingParams` in EarlybirdTen…
Apr 5, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
[cr-mixer/home-mixer] Remove getLinearRankingParams in EarlybirdTen… 
…sorflowBasedSimilarityEngine

Remove unused ranking params which are specified by services when making an Earlybird relevance search.

For cr-mixer: since we always set useTensorflowRanking = true in EarlybirdSimilarityEngineRouter, we will only ever use the TensorFlowBasedScoringFunction for ranking search results. That function doesn't rely on any of the linear params specified in getLinearRankingParams, nor the boosts because we set applyBoosts = false in the request. These parameters are therefore strictly redundant.

The parameters in home-mixer can be removed for essentially the same reason—the parameters are redundant given that we use the Tensorflow scoring function and don't apply boosts.
  • Loading branch information
twitter-team committed Apr 5, 2023
commit 138bb519975407d4ea0dc1478d897d451ef05dab
43 changes: 5 additions & 38 deletions ...ala/com/twitter/cr_mixer/similarity_engine/EarlybirdTensorflowBasedSimilarityEngine.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,6 @@ import com.twitter.search.earlybird.thriftscala.EarlybirdService
import com.twitter.search.earlybird.thriftscala.ThriftSearchQuery
import com.twitter.util.Time
import com.twitter.search.common.query.thriftjava.thriftscala.CollectorParams
import com.twitter.search.common.ranking.thriftscala.ThriftAgeDecayRankingParams
import com.twitter.search.common.ranking.thriftscala.ThriftLinearFeatureRankingParams
import com.twitter.search.common.ranking.thriftscala.ThriftRankingParams
import com.twitter.search.common.ranking.thriftscala.ThriftScoringFunctionType
import com.twitter.search.earlybird.thriftscala.ThriftSearchRelevanceOptions
Expand Down Expand Up @@ -97,7 +95,7 @@ object EarlybirdTensorflowBasedSimilarityEngine {
// Whether to collect conversation IDs. Remove it for now.
// collectConversationId = Gate.True(), // true for Home
rankingMode = ThriftSearchRankingMode.Relevance,
relevanceOptions = Some(getRelevanceOptions(query.useTensorflowRanking)),
relevanceOptions = Some(getRelevanceOptions),
collectorParams = Some(
CollectorParams(
// numResultsToReturn defines how many results each EB shard will return to search root
Expand All @@ -116,13 +114,11 @@ object EarlybirdTensorflowBasedSimilarityEngine {
// The specific values of recap relevance/reranking options correspond to
// experiment: enable_recap_reranking_2988,timeline_internal_disable_recap_filter
// bucket : enable_rerank,disable_filter
private def getRelevanceOptions(useTensorflowRanking: Boolean): ThriftSearchRelevanceOptions = {
private def getRelevanceOptions: ThriftSearchRelevanceOptions = {
ThriftSearchRelevanceOptions(
proximityScoring = true,
maxConsecutiveSameUser = Some(2),
rankingParams =
if (useTensorflowRanking) Some(getTensorflowBasedRankingParams)
else Some(getLinearRankingParams),
rankingParams = Some(getTensorflowBasedRankingParams),
maxHitsToProcess = Some(500),
maxUserBlendCount = Some(3),
proximityPhraseWeight = 9.0,
Expand All @@ -131,41 +127,12 @@ object EarlybirdTensorflowBasedSimilarityEngine {
}

private def getTensorflowBasedRankingParams: ThriftRankingParams = {
getLinearRankingParams.copy(
ThriftRankingParams(
`type` = Some(ThriftScoringFunctionType.TensorflowBased),
selectedTensorflowModel = Some("timelines_rectweet_replica"),
minScore = -1.0e100,
applyBoosts = false,
authorSpecificScoreAdjustments = None
)
}

private def getLinearRankingParams: ThriftRankingParams = {
ThriftRankingParams(
`type` = Some(ThriftScoringFunctionType.Linear),
minScore = -1.0e100,
retweetCountParams = Some(ThriftLinearFeatureRankingParams(weight = 20.0)),
replyCountParams = Some(ThriftLinearFeatureRankingParams(weight = 1.0)),
reputationParams = Some(ThriftLinearFeatureRankingParams(weight = 0.2)),
luceneScoreParams = Some(ThriftLinearFeatureRankingParams(weight = 2.0)),
textScoreParams = Some(ThriftLinearFeatureRankingParams(weight = 0.18)),
urlParams = Some(ThriftLinearFeatureRankingParams(weight = 2.0)),
isReplyParams = Some(ThriftLinearFeatureRankingParams(weight = 1.0)),
favCountParams = Some(ThriftLinearFeatureRankingParams(weight = 30.0)),
langEnglishUIBoost = 0.5,
langEnglishTweetBoost = 0.2,
langDefaultBoost = 0.02,
unknownLanguageBoost = 0.05,
offensiveBoost = 0.1,
inTrustedCircleBoost = 3.0,
multipleHashtagsOrTrendsBoost = 0.6,
inDirectFollowBoost = 4.0,
tweetHasTrendBoost = 1.1,
selfTweetBoost = 2.0,
tweetHasImageUrlBoost = 2.0,
tweetHasVideoUrlBoost = 2.0,
useUserLanguageInfo = true,
ageDecayParams = Some(ThriftAgeDecayRankingParams(slope = 0.005, base = 1.0))
)
}

}
22 changes: 0 additions & 22 deletions ...xer/server/src/main/scala/com/twitter/home_mixer/util/earlybird/RelevanceSearchUtil.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,28 +15,6 @@ object RelevanceSearchUtil {
`type` = Some(scr.ThriftScoringFunctionType.TensorflowBased),
selectedTensorflowModel = Some("timelines_rectweet_replica"),
minScore = -1.0e100,
retweetCountParams = Some(scr.ThriftLinearFeatureRankingParams(weight = 20.0)),
replyCountParams = Some(scr.ThriftLinearFeatureRankingParams(weight = 1.0)),
reputationParams = Some(scr.ThriftLinearFeatureRankingParams(weight = 0.2)),
luceneScoreParams = Some(scr.ThriftLinearFeatureRankingParams(weight = 2.0)),
textScoreParams = Some(scr.ThriftLinearFeatureRankingParams(weight = 0.18)),
urlParams = Some(scr.ThriftLinearFeatureRankingParams(weight = 2.0)),
isReplyParams = Some(scr.ThriftLinearFeatureRankingParams(weight = 1.0)),
favCountParams = Some(scr.ThriftLinearFeatureRankingParams(weight = 30.0)),
langEnglishUIBoost = 0.5,
langEnglishTweetBoost = 0.2,
langDefaultBoost = 0.02,
unknownLanguageBoost = 0.05,
offensiveBoost = 0.1,
inTrustedCircleBoost = 3.0,
multipleHashtagsOrTrendsBoost = 0.6,
inDirectFollowBoost = 4.0,
tweetHasTrendBoost = 1.1,
selfTweetBoost = 2.0,
tweetHasImageUrlBoost = 2.0,
tweetHasVideoUrlBoost = 2.0,
useUserLanguageInfo = true,
ageDecayParams = Some(scr.ThriftAgeDecayRankingParams(slope = 0.005, base = 1.0)),
selectedModels = Some(Map("home_mixer_unified_engagement_prod" -> 1.0)),
applyBoosts = false,
)
Expand Down