Info:: llama-server en tant que service

De wiki_pcwiki2
Aller à la navigationAller à la recherche

KEYWORDS: IA, LLAMA-SERVER, LLAMA.CPP, SRV18

Le 20/08/2025: llama-server en tant que service
NB: Attention, il faut compiler llamma-cpp.fresh

Pour lancer llama-server en tant que service: vi /etc/systemd/system/llama-server.service:
[Unit]
Description=Llama Server
After=network.target

[Service]
Type=simple
User=root
WorkingDirectory=/usr/local/IA
ExecStart=/bin/bash /usr/local/IA/llama-server.sh

Restart=on-failure
RestartSec=5

[Install]
WantedBy=multi-user.target

llama-server.sh:

#!/bin/bash
# lun. 18 août 2025 06:51:28 CEST
# LLM Disponibles:

  # 4G llama-2-7b-chat.Q4_K_M.gguf

  # 4G codellama-7b-instruct.Q4_K_M.gguf
  # 4G codellama-7b-instruct.Q4_K_S.gguf
  # 4G codellama-7b-instruct.Q5_0.gguf
  # 4G codellama-7b-instruct.Q5_K_M.gguf
  # 4G codellama-7b-instruct.Q5_K_S.gguf
  # 7G codellama-7b-instruct.Q8_0.gguf

  # 4G deepseek-coder-6.7b-instruct.Q4_0.gguf
  # 4G deepseek-coder-6.7b-instruct.Q5_0.gguf
  # 4G deepseek-coder-6.7b-instruct.Q5_K_M.gguf
  # 7G deepseek-coder-6.7b-instruct.Q8_0.gguf

  # 7G codegemma-7b-NES.Q6_K.gguf

 # 11G starcoder2-15b-instruct-v0.1-Q5_K_M.gguf

 # 11G gpt-oss-20b-32x2.4B-Q4_K_S.gguf
 # 11G gpt-oss-20b-Q4_K_M.gguf
 # 11G gpt-oss-20b-Q5_K_M.gguf

 # 11G Intel_gpt-oss-20b-gguf-q4ks-AutoRound_gpt-oss-20b-32x2.4B-Q4_K_S.gguf

 # 13G pygmalion-2-13b.Q8_0.gguf

 # 13G codestral-22b-v0.1-q4_k_m.gguf

 # 13G ggml-model-f16.gguf
 # 15G q_model_q5_0.gguf
 # 19G phind-codellama-34b-v2.Q4_K_M.gguf
 # 19G codellama-34b-instruct.Q4_K_M.gguf
 # 19G deepseek-coder-33b-instruct.Q4_K_M.gguf
 # 22G codellama-34b-instruct.Q5_K_S.gguf
 # 23G codellama-34b-instruct.Q5_K_M.gguf
 # 25G mixtral-8x7b-instruct-v0.1.Q4_K_M.gguf
 # 26G codellama-34b-instruct.Q6_K.gguf

cd /var/log/LLAMA-SERVER
if test $? -ne 0
then
    exit 1
fi

killall llama-server
strDir="/usr/local/IA/GGUF"

strModelLLM="pygmalion-2-13b.Q8_0.gguf"
strModelLLM="usr/local/codellama-7b-instruct.Q8_0.gguf"
strModelLLM="deepseek-coder-6.7b-instruct.Q8_0.gguf"

strLLM="${strDir}/${strModelLLM}"

echo "le modèle qui va se charger est: strLLM=${strLLM}"

# Utiliser la compilation de llama-server-fresh du 16/08/2025 (avec CUDA):
pcversionning -ff -n 10 /var/log/LLAMA-SERVER/

echo "les logs seront dans '/var/log/LLAMA-SERVER/llama-server.log'"

/usr/local/bin/llama-server 	    \
	--model "${strLLM}"             \
	--host 0.0.0.0 		            \
	--port 8080 		            \
	--ctx-size 4096 	            \
	--n-gpu-layers 16 	            \
	--threads 12 		            \
	--threads-batch 12 	            \
	--batch-size 32 	            \
	--ubatch-size 32 	            \
	--n-predict 4096 	            \
	--temp 0.1 		                \
	--top-k 40 		                \
	--top-p 0.9 		            \
	--parallel 4 		            \
	--cont-batching                 \
    --repeat-penalty 1.1            \
    > /var/log/LLAMA-SERVER/llama-server.log 2>&1

# --system-prompt 'Tu es un expert C++ sous Debian 12, spécialisé dans les standards modernes C++11. Utilise la librairie boost 1.83, les smart pointers, auto, range-based loops, et les algorithmes STL. Privilégie la sécurité mémoire et les performances.'

let intRet = $?
echo $intRet
if test $intRet -ne 0
then
    echo "Echec (erreur $intRet)"
fi

ls -alrt
echo "-------------------------------------------------------------"
# tail --lines 1000 -f /var/log/LLAMA-SERVER/llama-server.log