From e7fa2e06f8fcbace88e608c0ddb8f41b92193c44 Mon Sep 17 00:00:00 2001 From: Diego <38375572+diego-minguzzi@users.noreply.github.com> Date: Mon, 6 Nov 2023 19:07:46 +0100 Subject: [PATCH 1/3] Fixes the bug 1196 (#1232) * Current state of the branch. * Now gRPC is build only when the BUILD_GRPC_FOR_BACKEND_LLAMA variable is defined. * Now the local compilation of gRPC is executed on BUILD_GRPC_FOR_BACKEND_LLAMA. * Revised the Makefile. * Removed replace directives in go.mod. --------- Signed-off-by: Diego <38375572+diego-minguzzi@users.noreply.github.com> Co-authored-by: lunamidori5 <118759930+lunamidori5@users.noreply.github.com> Co-authored-by: Ettore Di Giacinto --- Makefile | 28 ++++++++- backend/cpp/grpc/.gitignore | 3 + backend/cpp/grpc/script/build_grpc.sh | 81 +++++++++++++++++++++++++++ 3 files changed, 110 insertions(+), 2 deletions(-) create mode 100644 backend/cpp/grpc/.gitignore create mode 100755 backend/cpp/grpc/script/build_grpc.sh diff --git a/Makefile b/Makefile index 758f1a0d..056c9ec6 100644 --- a/Makefile +++ b/Makefile @@ -267,6 +267,9 @@ clean: ## Remove build related file rm -rf ./go-piper rm -rf $(BINARY_NAME) rm -rf release/ + rm -rf ./backend/cpp/grpc/grpc_repo + rm -rf ./backend/cpp/grpc/build + rm -rf ./backend/cpp/grpc/installed_packages $(MAKE) -C backend/cpp/llama clean ## Build: @@ -409,9 +412,30 @@ ifeq ($(BUILD_TYPE),metal) cp go-llama/build/bin/ggml-metal.metal backend-assets/grpc/ endif -backend/cpp/llama/grpc-server: - LLAMA_VERSION=$(CPPLLAMA_VERSION) $(MAKE) -C backend/cpp/llama grpc-server +## BACKEND CPP LLAMA START +# Sets the variables in case it has to build the gRPC locally. +INSTALLED_PACKAGES=$(CURDIR)/backend/cpp/grpc/installed_packages +INSTALLED_LIB_CMAKE=$(INSTALLED_PACKAGES)/lib/cmake +ADDED_CMAKE_ARGS=-Dabsl_DIR=${INSTALLED_LIB_CMAKE}/absl \ + -DProtobuf_DIR=${INSTALLED_LIB_CMAKE}/protobuf \ + -Dutf8_range_DIR=${INSTALLED_LIB_CMAKE}/utf8_range \ + -DgRPC_DIR=${INSTALLED_LIB_CMAKE}/grpc \ + -DCMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES=${INSTALLED_PACKAGES}/include +backend/cpp/llama/grpc-server: +ifdef BUILD_GRPC_FOR_BACKEND_LLAMA + backend/cpp/grpc/script/build_grpc.sh ${INSTALLED_PACKAGES} + export _PROTOBUF_PROTOC=${INSTALLED_PACKAGES}/bin/proto && \ + export _GRPC_CPP_PLUGIN_EXECUTABLE=${INSTALLED_PACKAGES}/bin/grpc_cpp_plugin && \ + export PATH=${PATH}:${INSTALLED_PACKAGES}/bin && \ + CMAKE_ARGS="${ADDED_CMAKE_ARGS}" LLAMA_VERSION=$(CPPLLAMA_VERSION) $(MAKE) -C backend/cpp/llama grpc-server +else + echo "BUILD_GRPC_FOR_BACKEND_LLAMA is not defined." + LLAMA_VERSION=$(CPPLLAMA_VERSION) $(MAKE) -C backend/cpp/llama grpc-server +endif +## BACKEND CPP LLAMA END + +## backend-assets/grpc/llama-cpp: backend-assets/grpc backend/cpp/llama/grpc-server cp -rfv backend/cpp/llama/grpc-server backend-assets/grpc/llama-cpp # TODO: every binary should have its own folder instead, so can have different metal implementations diff --git a/backend/cpp/grpc/.gitignore b/backend/cpp/grpc/.gitignore new file mode 100644 index 00000000..e533db2e --- /dev/null +++ b/backend/cpp/grpc/.gitignore @@ -0,0 +1,3 @@ +installed_packages/ +grpc_build/ +grpc_repo/ diff --git a/backend/cpp/grpc/script/build_grpc.sh b/backend/cpp/grpc/script/build_grpc.sh new file mode 100755 index 00000000..98ece5b1 --- /dev/null +++ b/backend/cpp/grpc/script/build_grpc.sh @@ -0,0 +1,81 @@ +#!/bin/bash + +# Builds locally from sources the packages needed by the llama cpp backend. + +# Makes sure a few base packages exist. +# sudo apt-get --no-upgrade -y install g++ gcc binutils cmake git build-essential autoconf libtool pkg-config + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +echo "Script directory: $SCRIPT_DIR" + +CPP_INSTALLED_PACKAGES_DIR=$1 +if [ -z ${CPP_INSTALLED_PACKAGES_DIR} ]; then + echo "CPP_INSTALLED_PACKAGES_DIR env variable not set. Don't know where to install: failed."; + echo + exit -1 +fi + +if [ -d "${CPP_INSTALLED_PACKAGES_DIR}" ]; then + echo "gRPC installation directory already exists. Nothing to do." + exit 0 +fi + +# The depth when cloning a git repo. 1 speeds up the clone when the repo history is not needed. +GIT_CLONE_DEPTH=1 + +NUM_BUILD_THREADS=$(nproc --ignore=1) + +# Google gRPC -------------------------------------------------------------------------------------- +TAG_LIB_GRPC="v1.59.0" +GIT_REPO_LIB_GRPC="https://github.com/grpc/grpc.git" +GRPC_REPO_DIR="${SCRIPT_DIR}/../grpc_repo" +GRPC_BUILD_DIR="${SCRIPT_DIR}/../grpc_build" +SRC_DIR_LIB_GRPC="${GRPC_REPO_DIR}/grpc" + +echo "SRC_DIR_LIB_GRPC: ${SRC_DIR_LIB_GRPC}" +echo "GRPC_REPO_DIR: ${GRPC_REPO_DIR}" +echo "GRPC_BUILD_DIR: ${GRPC_BUILD_DIR}" + +mkdir -pv ${GRPC_REPO_DIR} + +rm -rf ${GRPC_BUILD_DIR} +mkdir -pv ${GRPC_BUILD_DIR} + +mkdir -pv ${CPP_INSTALLED_PACKAGES_DIR} + +if [ -d "${SRC_DIR_LIB_GRPC}" ]; then + echo "gRPC source already exists locally. Not cloned again." +else + ( cd ${GRPC_REPO_DIR} && \ + git clone --depth ${GIT_CLONE_DEPTH} -b ${TAG_LIB_GRPC} ${GIT_REPO_LIB_GRPC} && \ + cd ${SRC_DIR_LIB_GRPC} && \ + git submodule update --init --recursive --depth ${GIT_CLONE_DEPTH} + ) +fi + +( cd ${GRPC_BUILD_DIR} && \ + cmake -G "Unix Makefiles" \ + -DCMAKE_BUILD_TYPE=Release \ + -DgRPC_INSTALL=ON \ + -DEXECUTABLE_OUTPUT_PATH=${CPP_INSTALLED_PACKAGES_DIR}/grpc/bin \ + -DLIBRARY_OUTPUT_PATH=${CPP_INSTALLED_PACKAGES_DIR}/grpc/lib \ + -DgRPC_BUILD_TESTS=OFF \ + -DgRPC_BUILD_CSHARP_EXT=OFF \ + -DgRPC_BUILD_GRPC_CPP_PLUGIN=ON \ + -DgRPC_BUILD_GRPC_CSHARP_PLUGIN=OFF \ + -DgRPC_BUILD_GRPC_NODE_PLUGIN=OFF \ + -DgRPC_BUILD_GRPC_OBJECTIVE_C_PLUGIN=OFF \ + -DgRPC_BUILD_GRPC_PHP_PLUGIN=OFF \ + -DgRPC_BUILD_GRPC_PYTHON_PLUGIN=ON \ + -DgRPC_BUILD_GRPC_RUBY_PLUGIN=OFF \ + -Dprotobuf_WITH_ZLIB=ON \ + -DRE2_BUILD_TESTING=OFF \ + -DCMAKE_INSTALL_PREFIX=${CPP_INSTALLED_PACKAGES_DIR}/ \ + ${SRC_DIR_LIB_GRPC} && \ + cmake --build . -- -j ${NUM_BUILD_THREADS} && \ + cmake --build . --target install -- -j ${NUM_BUILD_THREADS} +) + +rm -rf ${GRPC_BUILD_DIR} +rm -rf ${GRPC_REPO_DIR} + From 562ac62f59b55c831207a908f07144473c2c4599 Mon Sep 17 00:00:00 2001 From: "LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Tue, 7 Nov 2023 08:37:55 +0100 Subject: [PATCH 2/3] :arrow_up: Update ggerganov/llama.cpp (#1242) Signed-off-by: GitHub Co-authored-by: mudler --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 056c9ec6..acba7b5e 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ GOLLAMA_VERSION?=aeba71ee842819da681ea537e78846dc75949ac0 GOLLAMA_STABLE_VERSION?=50cee7712066d9e38306eccadcfbb44ea87df4b7 -CPPLLAMA_VERSION?=629f917cd6b96ba1274c49a8aab163b1b189229d +CPPLLAMA_VERSION?=2833a6f63c1b87c7f4ac574bcf7a15a2f3bf3ede # gpt4all version GPT4ALL_REPO?=https://github.com/nomic-ai/gpt4all From 4e16bc2f13f059991656402da2eea1b53a201436 Mon Sep 17 00:00:00 2001 From: "LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Wed, 8 Nov 2023 08:21:12 +0100 Subject: [PATCH 3/3] :arrow_up: Update ggerganov/llama.cpp (#1256) Signed-off-by: GitHub Co-authored-by: mudler --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index acba7b5e..0bc262ee 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ GOLLAMA_VERSION?=aeba71ee842819da681ea537e78846dc75949ac0 GOLLAMA_STABLE_VERSION?=50cee7712066d9e38306eccadcfbb44ea87df4b7 -CPPLLAMA_VERSION?=2833a6f63c1b87c7f4ac574bcf7a15a2f3bf3ede +CPPLLAMA_VERSION?=0a7c980b6f94a049cb804573df2d8092a34df8e4 # gpt4all version GPT4ALL_REPO?=https://github.com/nomic-ai/gpt4all