Merge pull request #59 from junjihashimoto/feature/llm.c-train

Add train_gpt2 of llm.c for wgsl

experimental/kernels/Makefile

@@ -30,6 +30,12 @@ run: ./build/$(TARGET) dawnlib
 
 run_llm.c: ./build/test_gpt2 dawnlib
  $(LIBSPEC) && $<
+
+run_llm.c_train: ./build/train_gpt2 dawnlib
+ if [ ! -d dev ] ; then ln -s ./llm.c/dev ; fi
+ if [ ! -f gpt2_tokenizer.bin ] ; then ln -s ./llm.c/gpt2_tokenizer.bin ; fi
+ $(LIBSPEC) && $<
+
 llm.c:
  if [ ! -d llm.c ]; then git clone [email protected]:karpathy/llm.c.git ; fi
 
@@ -59,6 +65,26 @@ build/test_gpt2: llm.c build/unittest_kernels.o gpt2_124M.bin
  grep -q "^#include \"unittest_kernels.h\"" llm.c/train_gpt2.c || sed -i '1i#include \"unittest_kernels.h\"' llm.c/train_gpt2.c
  $(CC) $(CFLAGS) $(LDFLAGS) -o $@ llm.c/test_gpt2.c build/unittest_kernels.o
 
+build/train_gpt2: llm.c build/unittest_kernels.o gpt2_124M.bin
+ mkdir -p build
+ sed -i -e 's/void encoder_forward(/void ENCODER_FORWARD_CPU(/g' llm.c/train_gpt2.c
+ sed -i -e 's/void layernorm_forward(/void LAYERNORM_FORWARD_CPU(/g' llm.c/train_gpt2.c
+ sed -i -e 's/void matmul_forward(/void MATMUL_FORWARD_CPU(/g' llm.c/train_gpt2.c
+ sed -i -e 's/void attention_forward(/void ATTENTION_FORWARD_CPU(/g' llm.c/train_gpt2.c
+ sed -i -e 's/void gelu_forward(/void GELU_FORWARD_CPU(/g' llm.c/train_gpt2.c
+ sed -i -e 's/void residual_forward(/void RESIDUAL_FORWARD_CPU(/g' llm.c/train_gpt2.c
+ sed -i -e 's/void softmax_forward(/void SOFTMAX_FORWARD_CPU(/g' llm.c/train_gpt2.c
+ sed -i -e 's/void crossentropy_forward(/void CROSSENTROPY_FORWARD_CPU(/g' llm.c/train_gpt2.c
+ sed -i -e 's/void encoder_backward(/void ENCODER_BACKWARD_CPU(/g' llm.c/train_gpt2.c
+ sed -i -e 's/void layernorm_backward(/void LAYERNORM_BACKWARD_CPU(/g' llm.c/train_gpt2.c
+ sed -i -e 's/void matmul_backward(/void MATMUL_BACKWARD_CPU(/g' llm.c/train_gpt2.c
+ sed -i -e 's/void attention_backward(/void ATTENTION_BACKWARD_CPU(/g' llm.c/train_gpt2.c
+ sed -i -e 's/void gelu_backward(/void GELU_BACKWARD_CPU(/g' llm.c/train_gpt2.c
+ sed -i -e 's/void residual_backward(/void RESIDUAL_BACKWARD_CPU(/g' llm.c/train_gpt2.c
+ sed -i -e 's/void crossentropy_softmax_backward(/void CROSSENTROPY_SOFTMAX_BACKWARD_CPU(/g' llm.c/train_gpt2.c
+ grep -q "^#include \"unittest_kernels.h\"" llm.c/train_gpt2.c || sed -i '1i#include \"unittest_kernels.h\"' llm.c/train_gpt2.c
+ $(CC) $(CFLAGS) $(LDFLAGS) -o $@ llm.c/train_gpt2.c build/unittest_kernels.o
+
 build/unittest_kernels.o: unittest_llmc/unittest_kernels.cpp unittest_llmc/unittest_kernels.h kernels.h
  mkdir -p build && $(CXX) $(CXXFLAGS) -DNDEBUG -c -o $@ $<
 

experimental/kernels/unittest_llmc/unittest_kernels.cpp

@@ -1,4 +1,4 @@
-#include "gpu.h"
+#include "gpu.hpp"
 #include <array>
 #include <cstdio>
 #include <future>