Correctly initializing the logit scale parameter

cf. openai#46
wangjunxiao · Mar 22, 2021 · 43c953e · 43c953e
1 parent beba48f
commit 43c953e
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/clip/model.py b/clip/model.py
@@ -287,7 +287,7 @@ def __init__(self,
         self.ln_final = LayerNorm(transformer_width)
 
         self.text_projection = nn.Parameter(torch.empty(transformer_width, embed_dim))
-        self.logit_scale = nn.Parameter(torch.ones([]))
+        self.logit_scale = nn.Parameter(torch.FloatTensor([np.log(1/0.07)]))
 
         self.initialize_parameters()