|
441 | 441 | " if model == \"gpt-3.5-turbo\":\n",
|
442 | 442 | " print(\"Warning: gpt-3.5-turbo may change over time. Returning num tokens assuming gpt-3.5-turbo-0301.\")\n",
|
443 | 443 | " return num_tokens_from_messages(messages, model=\"gpt-3.5-turbo-0301\")\n",
|
| 444 | + " elif model == \"gpt-3.5-turbo-16k\":\n", |
| 445 | + " print(\"Warning: gpt-3.5-turbo-16k may change over time. Returning num tokens assuming gpt-3.5-turbo-16k-0613.\")\n", |
| 446 | + " return num_tokens_from_messages(messages, model=\"gpt-3.5-turbo-16k-0613\")\n", |
444 | 447 | " elif model == \"gpt-4\":\n",
|
445 | 448 | " print(\"Warning: gpt-4 may change over time. Returning num tokens assuming gpt-4-0314.\")\n",
|
446 | 449 | " return num_tokens_from_messages(messages, model=\"gpt-4-0314\")\n",
|
447 | 450 | " elif model == \"gpt-3.5-turbo-0301\":\n",
|
448 | 451 | " tokens_per_message = 4 # every message follows <|start|>{role/name}\\n{content}<|end|>\\n\n",
|
449 | 452 | " tokens_per_name = -1 # if there's a name, the role is omitted\n",
|
450 |
| - " elif model == \"gpt-4-0314\":\n", |
| 453 | + " elif model in {\"gpt-4-0314\", \"gpt-3.5-turbo-0613\", \"gpt-3.5-turbo-16k-0613\"}:\n", |
451 | 454 | " tokens_per_message = 3\n",
|
452 | 455 | " tokens_per_name = 1\n",
|
453 | 456 | " else:\n",
|
|
0 commit comments