Ft llama opt

.gitignore

@@ -15,4 +15,7 @@ __pycache__/
 **/.ipynb_checkpoints/
 
 /3rdparty/NeMo/
-/3rdparty/apex/
+/3rdparty/apex/
+20B_checkpoints/
+compile_commands.json
+model/

CMakeLists.txt

@@ -418,7 +418,7 @@ add_library(transformer-shared SHARED
 
 if (BUILD_MULTI_GPU)
 target_link_libraries(transformer-shared PUBLIC
-  -lmpi
+  -lmpi -lmpi_cxx
   ${NCCL_LIBRARIES}
 )
 endif()

examples/cpp/CMakeLists.txt

@@ -27,6 +27,7 @@ add_subdirectory(wenet)
 add_subdirectory(gptj)
 add_subdirectory(gptneox)
 add_subdirectory(multi_gpu_gpt)
+#add_subdirectory(llama)
 
 if(ENABLE_FP8)
     add_subdirectory(gpt_fp8)

examples/cpp/llama/CMakeLists.txt

@@ -0,0 +1,22 @@
+# Copyright (c) 2019-2023, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+add_library(llama_example_utils STATIC llama_example_utils.cc)
+target_link_libraries(llama_example_utils PUBLIC -lcublas -lcublasLt -lcudart
+                      nvtx_utils mpi_utils nccl_utils)
+
+add_executable(llama_example llama_example.cc)
+target_link_libraries(llama_example PUBLIC -lcublas -lcublasLt -lcudart
+    LLaMA mpi_utils nccl_utils nvtx_utils 
+    llama_example_utils word_list)

examples/cpp/llama/bad_words.csv

@@ -0,0 +1,2 @@
+7768,3908
+1,2

examples/cpp/llama/llama_config.ini

@@ -0,0 +1,21 @@
+[ft_instance_hyperparameter]
+model_name=llama_33B
+model_dir=../models/llama
+data_type=fp16
+pipeline_para_size=4
+
+
+[request]
+request_batch_size=32
+start_pos=2
+
+[llama_33B]
+head_num=52
+size_per_head=128
+vocab_size=32000
+decoder_layers=60
+rotary_embedding=128
+multiple_of=256
+max_seq_len=1024
+padding_id=0
+random_seed=0