Skip to content

Commit cea5a0e

Browse files
mudlermudler
authored
feat(template): read jinja templates from gguf files (#4332)
* Read jinja templates as fallback Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Move templating out of model loader Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Test TemplateMessages Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Set role and content from transformers Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Tests: be more flexible Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * More jinja Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Small refactoring and adaptations Signed-off-by: Ettore Di Giacinto <mudler@localai.io> --------- Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
1 parent f5e1527 commit cea5a0e

23 files changed

+967
-781
lines changed

‎core/application.go

Lines changed: 0 additions & 38 deletions
This file was deleted.

‎core/application/application.go

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
package application
2+
3+
import (
4+
"github.com/mudler/LocalAI/core/config"
5+
"github.com/mudler/LocalAI/pkg/model"
6+
"github.com/mudler/LocalAI/pkg/templates"
7+
)
8+
9+
type Application struct {
10+
backendLoader *config.BackendConfigLoader
11+
modelLoader *model.ModelLoader
12+
applicationConfig *config.ApplicationConfig
13+
templatesEvaluator *templates.Evaluator
14+
}
15+
16+
func newApplication(appConfig *config.ApplicationConfig) *Application {
17+
return &Application{
18+
backendLoader: config.NewBackendConfigLoader(appConfig.ModelPath),
19+
modelLoader: model.NewModelLoader(appConfig.ModelPath),
20+
applicationConfig: appConfig,
21+
templatesEvaluator: templates.NewEvaluator(appConfig.ModelPath),
22+
}
23+
}
24+
25+
func (a *Application) BackendLoader() *config.BackendConfigLoader {
26+
return a.backendLoader
27+
}
28+
29+
func (a *Application) ModelLoader() *model.ModelLoader {
30+
return a.modelLoader
31+
}
32+
33+
func (a *Application) ApplicationConfig() *config.ApplicationConfig {
34+
return a.applicationConfig
35+
}
36+
37+
func (a *Application) TemplatesEvaluator() *templates.Evaluator {
38+
return a.templatesEvaluator
39+
}

‎core/startup/config_file_watcher.go renamed to ‎core/application/config_file_watcher.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package startup
1+
package application
22

33
import (
44
"encoding/json"
@@ -8,8 +8,8 @@ import (
88
"path/filepath"
99
"time"
1010

11-
"github.com/fsnotify/fsnotify"
1211
"dario.cat/mergo"
12+
"github.com/fsnotify/fsnotify"
1313
"github.com/mudler/LocalAI/core/config"
1414
"github.com/rs/zerolog/log"
1515
)

‎core/startup/startup.go renamed to ‎core/application/startup.go

Lines changed: 23 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,25 @@
1-
package startup
1+
package application
22

33
import (
44
"fmt"
55
"os"
66

7-
"github.com/mudler/LocalAI/core"
87
"github.com/mudler/LocalAI/core/backend"
98
"github.com/mudler/LocalAI/core/config"
109
"github.com/mudler/LocalAI/core/services"
1110
"github.com/mudler/LocalAI/internal"
1211
"github.com/mudler/LocalAI/pkg/assets"
12+
1313
"github.com/mudler/LocalAI/pkg/library"
1414
"github.com/mudler/LocalAI/pkg/model"
1515
pkgStartup "github.com/mudler/LocalAI/pkg/startup"
1616
"github.com/mudler/LocalAI/pkg/xsysinfo"
1717
"github.com/rs/zerolog/log"
1818
)
1919

20-
func Startup(opts ...config.AppOption) (*config.BackendConfigLoader, *model.ModelLoader, *config.ApplicationConfig, error) {
20+
func New(opts ...config.AppOption) (*Application, error) {
2121
options := config.NewApplicationConfig(opts...)
22+
application := newApplication(options)
2223

2324
log.Info().Msgf("Starting LocalAI using %d threads, with models path: %s", options.Threads, options.ModelPath)
2425
log.Info().Msgf("LocalAI version: %s", internal.PrintableVersion())
@@ -36,68 +37,65 @@ func Startup(opts ...config.AppOption) (*config.BackendConfigLoader, *model.Mode
3637

3738
// Make sure directories exists
3839
if options.ModelPath == "" {
39-
return nil, nil, nil, fmt.Errorf("options.ModelPath cannot be empty")
40+
return nil, fmt.Errorf("options.ModelPath cannot be empty")
4041
}
4142
err = os.MkdirAll(options.ModelPath, 0750)
4243
if err != nil {
43-
return nil, nil, nil, fmt.Errorf("unable to create ModelPath: %q", err)
44+
return nil, fmt.Errorf("unable to create ModelPath: %q", err)
4445
}
4546
if options.ImageDir != "" {
4647
err := os.MkdirAll(options.ImageDir, 0750)
4748
if err != nil {
48-
return nil, nil, nil, fmt.Errorf("unable to create ImageDir: %q", err)
49+
return nil, fmt.Errorf("unable to create ImageDir: %q", err)
4950
}
5051
}
5152
if options.AudioDir != "" {
5253
err := os.MkdirAll(options.AudioDir, 0750)
5354
if err != nil {
54-
return nil, nil, nil, fmt.Errorf("unable to create AudioDir: %q", err)
55+
return nil, fmt.Errorf("unable to create AudioDir: %q", err)
5556
}
5657
}
5758
if options.UploadDir != "" {
5859
err := os.MkdirAll(options.UploadDir, 0750)
5960
if err != nil {
60-
return nil, nil, nil, fmt.Errorf("unable to create UploadDir: %q", err)
61+
return nil, fmt.Errorf("unable to create UploadDir: %q", err)
6162
}
6263
}
6364

6465
if err := pkgStartup.InstallModels(options.Galleries, options.ModelLibraryURL, options.ModelPath, options.EnforcePredownloadScans, nil, options.ModelsURL...); err != nil {
6566
log.Error().Err(err).Msg("error installing models")
6667
}
6768

68-
cl := config.NewBackendConfigLoader(options.ModelPath)
69-
ml := model.NewModelLoader(options.ModelPath)
70-
7169
configLoaderOpts := options.ToConfigLoaderOptions()
7270

73-
if err := cl.LoadBackendConfigsFromPath(options.ModelPath, configLoaderOpts...); err != nil {
71+
if err := application.BackendLoader().LoadBackendConfigsFromPath(options.ModelPath, configLoaderOpts...); err != nil {
7472
log.Error().Err(err).Msg("error loading config files")
7573
}
7674

7775
if options.ConfigFile != "" {
78-
if err := cl.LoadMultipleBackendConfigsSingleFile(options.ConfigFile, configLoaderOpts...); err != nil {
76+
if err := application.BackendLoader().LoadMultipleBackendConfigsSingleFile(options.ConfigFile, configLoaderOpts...); err != nil {
7977
log.Error().Err(err).Msg("error loading config file")
8078
}
8179
}
8280

83-
if err := cl.Preload(options.ModelPath); err != nil {
81+
if err := application.BackendLoader().Preload(options.ModelPath); err != nil {
8482
log.Error().Err(err).Msg("error downloading models")
8583
}
8684

8785
if options.PreloadJSONModels != "" {
8886
if err := services.ApplyGalleryFromString(options.ModelPath, options.PreloadJSONModels, options.EnforcePredownloadScans, options.Galleries); err != nil {
89-
return nil, nil, nil, err
87+
return nil, err
9088
}
9189
}
9290

9391
if options.PreloadModelsFromPath != "" {
9492
if err := services.ApplyGalleryFromFile(options.ModelPath, options.PreloadModelsFromPath, options.EnforcePredownloadScans, options.Galleries); err != nil {
95-
return nil, nil, nil, err
93+
return nil, err
9694
}
9795
}
9896

9997
if options.Debug {
100-
for _, v := range cl.GetAllBackendConfigs() {
98+
for _, v := range application.BackendLoader().GetAllBackendConfigs() {
10199
log.Debug().Msgf("Model: %s (config: %+v)", v.Name, v)
102100
}
103101
}
@@ -123,20 +121,20 @@ func Startup(opts ...config.AppOption) (*config.BackendConfigLoader, *model.Mode
123121
go func() {
124122
<-options.Context.Done()
125123
log.Debug().Msgf("Context canceled, shutting down")
126-
err := ml.StopAllGRPC()
124+
err := application.ModelLoader().StopAllGRPC()
127125
if err != nil {
128126
log.Error().Err(err).Msg("error while stopping all grpc backends")
129127
}
130128
}()
131129

132130
if options.WatchDog {
133131
wd := model.NewWatchDog(
134-
ml,
132+
application.ModelLoader(),
135133
options.WatchDogBusyTimeout,
136134
options.WatchDogIdleTimeout,
137135
options.WatchDogBusy,
138136
options.WatchDogIdle)
139-
ml.SetWatchDog(wd)
137+
application.ModelLoader().SetWatchDog(wd)
140138
go wd.Run()
141139
go func() {
142140
<-options.Context.Done()
@@ -147,25 +145,25 @@ func Startup(opts ...config.AppOption) (*config.BackendConfigLoader, *model.Mode
147145

148146
if options.LoadToMemory != nil {
149147
for _, m := range options.LoadToMemory {
150-
cfg, err := cl.LoadBackendConfigFileByName(m, options.ModelPath,
148+
cfg, err := application.BackendLoader().LoadBackendConfigFileByName(m, options.ModelPath,
151149
config.LoadOptionDebug(options.Debug),
152150
config.LoadOptionThreads(options.Threads),
153151
config.LoadOptionContextSize(options.ContextSize),
154152
config.LoadOptionF16(options.F16),
155153
config.ModelPath(options.ModelPath),
156154
)
157155
if err != nil {
158-
return nil, nil, nil, err
156+
return nil, err
159157
}
160158

161159
log.Debug().Msgf("Auto loading model %s into memory from file: %s", m, cfg.Model)
162160

163161
o := backend.ModelOptions(*cfg, options)
164162

165163
var backendErr error
166-
_, backendErr = ml.Load(o...)
164+
_, backendErr = application.ModelLoader().Load(o...)
167165
if backendErr != nil {
168-
return nil, nil, nil, err
166+
return nil, err
169167
}
170168
}
171169
}
@@ -174,7 +172,7 @@ func Startup(opts ...config.AppOption) (*config.BackendConfigLoader, *model.Mode
174172
startWatcher(options)
175173

176174
log.Info().Msg("core/startup process completed!")
177-
return cl, ml, options, nil
175+
return application, nil
178176
}
179177

180178
func startWatcher(options *config.ApplicationConfig) {
@@ -201,32 +199,3 @@ func startWatcher(options *config.ApplicationConfig) {
201199
log.Error().Err(err).Msg("failed creating watcher")
202200
}
203201
}
204-
205-
// In Lieu of a proper DI framework, this function wires up the Application manually.
206-
// This is in core/startup rather than core/state.go to keep package references clean!
207-
func createApplication(appConfig *config.ApplicationConfig) *core.Application {
208-
app := &core.Application{
209-
ApplicationConfig: appConfig,
210-
BackendConfigLoader: config.NewBackendConfigLoader(appConfig.ModelPath),
211-
ModelLoader: model.NewModelLoader(appConfig.ModelPath),
212-
}
213-
214-
var err error
215-
216-
// app.EmbeddingsBackendService = backend.NewEmbeddingsBackendService(app.ModelLoader, app.BackendConfigLoader, app.ApplicationConfig)
217-
// app.ImageGenerationBackendService = backend.NewImageGenerationBackendService(app.ModelLoader, app.BackendConfigLoader, app.ApplicationConfig)
218-
// app.LLMBackendService = backend.NewLLMBackendService(app.ModelLoader, app.BackendConfigLoader, app.ApplicationConfig)
219-
// app.TranscriptionBackendService = backend.NewTranscriptionBackendService(app.ModelLoader, app.BackendConfigLoader, app.ApplicationConfig)
220-
// app.TextToSpeechBackendService = backend.NewTextToSpeechBackendService(app.ModelLoader, app.BackendConfigLoader, app.ApplicationConfig)
221-
222-
app.BackendMonitorService = services.NewBackendMonitorService(app.ModelLoader, app.BackendConfigLoader, app.ApplicationConfig)
223-
app.GalleryService = services.NewGalleryService(app.ApplicationConfig)
224-
// app.OpenAIService = services.NewOpenAIService(app.ModelLoader, app.BackendConfigLoader, app.ApplicationConfig, app.LLMBackendService)
225-
226-
app.LocalAIMetricsService, err = services.NewLocalAIMetricsService()
227-
if err != nil {
228-
log.Error().Err(err).Msg("encountered an error initializing metrics service, startup will continue but metrics will not be tracked.")
229-
}
230-
231-
return app
232-
}

‎core/cli/run.go

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,12 @@ import (
66
"strings"
77
"time"
88

9+
"github.com/mudler/LocalAI/core/application"
910
cli_api "github.com/mudler/LocalAI/core/cli/api"
1011
cliContext "github.com/mudler/LocalAI/core/cli/context"
1112
"github.com/mudler/LocalAI/core/config"
1213
"github.com/mudler/LocalAI/core/http"
1314
"github.com/mudler/LocalAI/core/p2p"
14-
"github.com/mudler/LocalAI/core/startup"
1515
"github.com/rs/zerolog"
1616
"github.com/rs/zerolog/log"
1717
)
@@ -186,16 +186,16 @@ func (r *RunCMD) Run(ctx *cliContext.Context) error {
186186
}
187187

188188
if r.PreloadBackendOnly {
189-
_, _, _, err := startup.Startup(opts...)
189+
_, err := application.New(opts...)
190190
return err
191191
}
192192

193-
cl, ml, options, err := startup.Startup(opts...)
193+
app, err := application.New(opts...)
194194
if err != nil {
195195
return fmt.Errorf("failed basic startup tasks with error %s", err.Error())
196196
}
197197

198-
appHTTP, err := http.App(cl, ml, options)
198+
appHTTP, err := http.API(app)
199199
if err != nil {
200200
log.Error().Err(err).Msg("error during HTTP App construction")
201201
return err

‎core/config/backend_config.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -206,6 +206,8 @@ type TemplateConfig struct {
206206
JoinChatMessagesByCharacter *string `yaml:"join_chat_messages_by_character"`
207207

208208
Multimodal string `yaml:"multimodal"`
209+
210+
JinjaTemplate bool `yaml:"jinja_template"`
209211
}
210212

211213
func (c *BackendConfig) UnmarshalYAML(value *yaml.Node) error {

‎core/config/guesser.go

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,14 +26,14 @@ const (
2626
type settingsConfig struct {
2727
StopWords []string
2828
TemplateConfig TemplateConfig
29-
RepeatPenalty float64
29+
RepeatPenalty float64
3030
}
3131

3232
// default settings to adopt with a given model family
3333
var defaultsSettings map[familyType]settingsConfig = map[familyType]settingsConfig{
3434
Gemma: {
3535
RepeatPenalty: 1.0,
36-
StopWords: []string{"<|im_end|>", "<end_of_turn>", "<start_of_turn>"},
36+
StopWords: []string{"<|im_end|>", "<end_of_turn>", "<start_of_turn>"},
3737
TemplateConfig: TemplateConfig{
3838
Chat: "{{.Input }}\n<start_of_turn>model\n",
3939
ChatMessage: "<start_of_turn>{{if eq .RoleName \"assistant\" }}model{{else}}{{ .RoleName }}{{end}}\n{{ if .Content -}}\n{{.Content -}}\n{{ end -}}<end_of_turn>",
@@ -200,6 +200,18 @@ func guessDefaultsFromFile(cfg *BackendConfig, modelPath string) {
200200
} else {
201201
log.Debug().Any("family", family).Msgf("guessDefaultsFromFile: no template found for family")
202202
}
203+
204+
if cfg.HasTemplate() {
205+
return
206+
}
207+
208+
// identify from well known templates first, otherwise use the raw jinja template
209+
chatTemplate, found := f.Header.MetadataKV.Get("tokenizer.chat_template")
210+
if found {
211+
// try to use the jinja template
212+
cfg.TemplateConfig.JinjaTemplate = true
213+
cfg.TemplateConfig.ChatMessage = chatTemplate.ValueString()
214+
}
203215
}
204216

205217
func identifyFamily(f *gguf.GGUFFile) familyType {

0 commit comments

Comments
 (0)