diff --git a/dockerfile b/dockerfile
index b23f144..b884428 100644
--- a/dockerfile
+++ b/dockerfile
@@ -62,7 +62,7 @@ RUN Python_Version=$(echo "$Python_Release" | cut -d. -f1,2) && \
     ln -sf /usr/local/bin/pip$Python_Version /usr/bin/pip
 
 # Build pytorch from souce
-FROM build-python AS build-pytorch
+FROM build-python AS clone-pytorch
 WORKDIR /usr/src
 
 # Get release version from https://github.com/pytorch/pytorch/releases
@@ -75,6 +75,15 @@ ARG CUDA_ARCH_LIST="7.5;8.0;8.6;8.9"
 RUN git clone --recursive -b v${Pytorch_Release} https://github.com/pytorch/pytorch.git
 WORKDIR /usr/src/pytorch
 
+RUN python -m pip install numpy typing_extensions future sympy
+
+RUN pip install -r requirements.txt
+
+RUN git submodule sync && \
+    git submodule update --init --recursive
+
+FROM clone-pytorch AS build-pytorch
+
 ENV USE_CUDA=1 \
     USE_CUDNN=1 \
     BUILD_CAFFE2=0 \
@@ -92,13 +101,6 @@ ENV USE_CUDA=1 \
     CUDA_HOME="/usr/local/cuda" \
     USE_NVRTC=1
 
-RUN python -m pip install numpy typing_extensions future sympy
-
-RUN pip install -r requirements.txt
-
-RUN git submodule sync && \
-    git submodule update --init --recursive
-
 RUN MAX_JOBS=${Pytorch_Jobs} && \
     TORCH_CUDA_ARCH_LIST=${CUDA_ARCH_LIST} && \
     python setup.py install