Merge branch 'master' into update/CPPLLAMA_VERSION

Author: mudler

Makefile

@@ -54,7 +54,7 @@ override LD_FLAGS += -X "github.com/go-skynet/LocalAI/internal.Commit=$(shell gi
 
 OPTIONAL_TARGETS?=
 
-OS := $(shell uname -s)
+export OS := $(shell uname -s)
 ARCH := $(shell uname -m)
 GREEN  := $(shell tput -Txterm setaf 2)
 YELLOW := $(shell tput -Txterm setaf 3)
@@ -764,7 +764,7 @@ backend-assets/grpc/llama-cpp-grpc: backend-assets/grpc
 	cp -rf backend/cpp/llama backend/cpp/llama-grpc
 	$(MAKE) -C backend/cpp/llama-grpc purge
 	$(info ${GREEN}I llama-cpp build info:grpc${RESET})
-	CMAKE_ARGS="$(CMAKE_ARGS) -DLLAMA_RPC=ON -DLLAMA_AVX=off -DLLAMA_AVX2=off -DLLAMA_AVX512=off -DLLAMA_FMA=off -DLLAMA_F16C=off" $(MAKE) VARIANT="llama-grpc" build-llama-cpp-grpc-server
+	CMAKE_ARGS="$(CMAKE_ARGS) -DLLAMA_RPC=ON -DLLAMA_AVX=off -DLLAMA_AVX2=off -DLLAMA_AVX512=off -DLLAMA_FMA=off -DLLAMA_F16C=off" TARGET="--target grpc-server --target rpc-server" $(MAKE) VARIANT="llama-grpc" build-llama-cpp-grpc-server
 	cp -rfv backend/cpp/llama-grpc/grpc-server backend-assets/grpc/llama-cpp-grpc
 
 backend-assets/util/llama-cpp-rpc-server: backend-assets/grpc/llama-cpp-grpc

backend/cpp/llama/Makefile

@@ -4,6 +4,7 @@ LLAMA_VERSION?=
 CMAKE_ARGS?=
 BUILD_TYPE?=
 ONEAPI_VARS?=/opt/intel/oneapi/setvars.sh
+TARGET?=--target grpc-server
 
 # If build type is cublas, then we set -DLLAMA_CUBLAS=ON to CMAKE_ARGS automatically
 ifeq ($(BUILD_TYPE),cublas)
@@ -20,9 +21,11 @@ else ifeq ($(BUILD_TYPE),hipblas)
 	CMAKE_ARGS+=-DLLAMA_HIPBLAS=ON
 # If it's OSX, DO NOT embed the metal library - -DLLAMA_METAL_EMBED_LIBRARY=ON requires further investigation
 # But if it's OSX without metal, disable it here
-else ifeq ($(OS),darwin)
+else ifeq ($(OS),Darwin)
 	ifneq ($(BUILD_TYPE),metal)
 		CMAKE_ARGS+=-DLLAMA_METAL=OFF
+	else
+		TARGET+=--target ggml-metal
 	endif
 endif
 
@@ -62,8 +65,8 @@ grpc-server: llama.cpp llama.cpp/examples/grpc-server
 	@echo "Building grpc-server with $(BUILD_TYPE) build type and $(CMAKE_ARGS)"
 ifneq (,$(findstring sycl,$(BUILD_TYPE)))
 	bash -c "source $(ONEAPI_VARS); \
-	cd llama.cpp && mkdir -p build && cd build && cmake .. $(CMAKE_ARGS) && $(MAKE)"
+	cd llama.cpp && mkdir -p build && cd build && cmake .. $(CMAKE_ARGS) && cmake --build . --config Release $(TARGET)"
 else
-	cd llama.cpp && mkdir -p build && cd build && cmake .. $(CMAKE_ARGS) && $(MAKE)
+	cd llama.cpp && mkdir -p build && cd build && cmake .. $(CMAKE_ARGS) && cmake --build . --config Release $(TARGET)
 endif
 	cp llama.cpp/build/bin/grpc-server .

backend/cpp/llama/grpc-server.cpp

@@ -886,6 +886,8 @@ struct llama_server_context
             {"task_id", slot->task_id},
         });
 
+        LOG_TEE("sampling: \n%s\n", llama_sampling_print(slot->sparams).c_str());
+
         return true;
     }
 

core/backend/options.go

@@ -142,12 +142,14 @@ func gRPCPredictOpts(c config.BackendConfig, modelPath string) *pb.PredictOption
 		MirostatTAU:         float32(*c.LLMConfig.MirostatTAU),
 		Debug:               *c.Debug,
 		StopPrompts:         c.StopWords,
-		Repeat:              int32(c.RepeatPenalty),
+		Repeat:              int32(c.RepeatLastN),
+		FrequencyPenalty:    float32(c.FrequencyPenalty),
+		PresencePenalty:     float32(c.PresencePenalty),
+		Penalty:             float32(c.RepeatPenalty),
 		NKeep:               int32(c.Keep),
 		Batch:               int32(c.Batch),
 		IgnoreEOS:           c.IgnoreEOS,
 		Seed:                getSeed(c),
-		FrequencyPenalty:    float32(c.FrequencyPenalty),
 		MLock:               *c.MMlock,
 		MMap:                *c.MMap,
 		MainGPU:             c.MainGPU,

core/schema/prediction.go

@@ -25,7 +25,10 @@ type PredictionOptions struct {
 	Batch         int     `json:"batch" yaml:"batch"`
 	IgnoreEOS     bool    `json:"ignore_eos" yaml:"ignore_eos"`
 	RepeatPenalty float64 `json:"repeat_penalty" yaml:"repeat_penalty"`
-	Keep          int     `json:"n_keep" yaml:"n_keep"`
+
+	RepeatLastN int `json:"repeat_last_n" yaml:"repeat_last_n"`
+
+	Keep int `json:"n_keep" yaml:"n_keep"`
 
 	FrequencyPenalty float64  `json:"frequency_penalty" yaml:"frequency_penalty"`
 	PresencePenalty  float64  `json:"presence_penalty" yaml:"presence_penalty"`

pkg/functions/options.go

@@ -42,3 +42,9 @@ func SetPrefix(suffix string) func(*GrammarOption) {
 		o.Prefix = suffix
 	}
 }
+
+func SetPropOrder(order string) func(*GrammarOption) {
+	return func(o *GrammarOption) {
+		o.PropOrder = order
+	}
+}

pkg/functions/parse.go

@@ -32,6 +32,11 @@ type GrammarConfig struct {
 
 	// ExpectStringsAfterJSON enables mixed string suffix
 	ExpectStringsAfterJSON bool `yaml:"expect_strings_after_json"`
+
+	// PropOrder selects what order to print properties
+	// for instance name,arguments will make print { "name": "foo", "arguments": { "bar": "baz" } }
+	// instead of { "arguments": { "bar": "baz" }, "name": "foo" }
+	PropOrder string `yaml:"properties_order"`
 }
 
 // FunctionsConfig is the configuration for the tool/function call.
@@ -104,6 +109,8 @@ func (g GrammarConfig) Options() []func(o *GrammarOption) {
 	if g.ExpectStringsAfterJSON {
 		opts = append(opts, ExpectStringsAfterJSON)
 	}
+
+	opts = append(opts, SetPropOrder(g.PropOrder))
 	return opts
 }