[1.0] Added /opt/readies submodule (#426)

* Added submodule opt/readies

* Build system sysnc

Author: rafie

.circleci/config.yml

@@ -16,6 +16,9 @@ venv*/
 *.tar.gz
 /VARIANT
 
+# Docs API reference 
+docs/api_reference.md
+
 ### Cmake auto tools
 cmake-build-debug
 

.gitignore

@@ -0,0 +1,3 @@
+[submodule "opt/readies"]
+	path = opt/readies
+	url = https://github.com/RedisLabsModules/readies.git

.gitmodules

@@ -11,10 +11,20 @@ ARG OS=debian:buster-slim
 # ARCH=x64|arm64v8|arm32v7
 ARG ARCH=x64
 
+ARG PACK=0
+ARG TEST=0
+
 #----------------------------------------------------------------------------------------------
 FROM redisfab/redis:${REDIS_VER}-${ARCH}-${OSNICK} AS redis
 FROM ${OS} AS builder
 
+ARG OSNICK
+ARG OS
+ARG ARCH
+ARG REDIS_VER
+
+RUN echo "Building for ${OSNICK} (${OS}) for ${ARCH} [with Redis ${REDIS_VER}]"
+
 WORKDIR /build
 COPY --from=redis /usr/local/ /usr/local/
 
@@ -30,18 +40,29 @@ RUN if [ "$DEPS_ARGS" = "" ]; then ./get_deps.sh cpu; else env $DEPS_ARGS ./get_
 
 ARG BUILD_ARGS=""
 ADD ./ /build
-RUN make -C opt build $BUILD_ARGS SHOW=1
+RUN set -e ;\
+    . ./opt/readies/bin/sourced ./profile.d ;\
+    make -C opt build $BUILD_ARGS SHOW=1
 
-ARG PACK=0
-ARG TEST=0
+ARG PACK
+ARG TEST
 
+RUN mkdir -p bin/artifacts
 RUN if [ "$PACK" = "1" ]; then make -C opt pack; fi
-RUN if [ "$TEST" = "1" ]; then make -C opt test $BUILD_ARGS NO_LFS=1; fi
+
+RUN if [ "$TEST" = "1" ]; then TEST= make -C opt test $BUILD_ARGS NO_LFS=1; fi
 
 #----------------------------------------------------------------------------------------------
 FROM redisfab/redis:${REDIS_VER}-${ARCH}-${OSNICK}
 
-RUN set -e; apt-get -qq update; apt-get -q install -y libgomp1
+ARG OSNICK
+ARG OS
+ARG ARCH
+ARG REDIS_VER
+ARG PACK
+
+RUN if [ ! -z $(command -v apt-get) ]; then apt-get -qq update; apt-get -q install -y libgomp1; fi
+RUN if [ ! -z $(command -v yum) ]; then yum install -y libgomp; fi 
 
 ENV REDIS_MODULES /usr/lib/redis/modules
 ENV LD_LIBRARY_PATH $REDIS_MODULES
@@ -50,6 +71,8 @@ RUN mkdir -p $REDIS_MODULES/
 
 COPY --from=builder /build/install-cpu/ $REDIS_MODULES/
 
+COPY --from=builder /build/bin/artifacts/ /var/opt/redislabs/artifacts
+
 WORKDIR /data
 EXPOSE 6379
 CMD ["--loadmodule", "/usr/lib/redis/modules/redisai.so"]

Dockerfile

@@ -11,12 +11,24 @@ ARG OS=debian:buster-slim
 # ARCH=arm64v8|arm32v7
 ARG ARCH=arm64v8
 
+ARG PACK=0
+ARG TEST=0
+
 #----------------------------------------------------------------------------------------------
-FROM redisfab/redis-xbuild:${REDIS_VER}-${ARCH}-${OSNICK} AS builder
+FROM redisfab/redis:${REDIS_VER}-${ARCH}-${OSNICK} AS redis
+FROM redisfab/xbuild:${ARCH}-${OS} AS builder
+
+ARG OSNICK
+ARG OS
+ARG ARCH
+ARG REDIS_VER
 
 RUN [ "cross-build-start" ]
 
+RUN echo "Building for ${OSNICK} (${OS}) for ${ARCH}"
+
 WORKDIR /build
+COPY --from=redis /usr/local/ /usr/local/
 
 COPY ./opt/ opt/
 COPY ./test/test_requirements.txt test/
@@ -30,13 +42,15 @@ RUN if [ "$DEPS_ARGS" = "" ]; then ./get_deps.sh cpu; else env $DEPS_ARGS ./get_
 
 ARG BUILD_ARGS=""
 ADD ./ /build
-RUN make -C opt build $BUILD_ARGS SHOW=1
+RUN set -e ;\
+    . ./opt/readies/bin/sourced ./profile.d ;\
+    make -C opt build $BUILD_ARGS SHOW=1
 
-ARG PACK=0
-ARG TEST=0
+ARG PACK
+ARG TEST
 
 RUN if [ "$PACK" = "1" ]; then make -C opt pack; fi
-RUN if [ "$TEST" = "1" ]; then make -C opt test $BUILD_ARGS NO_LFS=1; fi
+RUN if [ "$TEST" = "1" ]; then TEST= make -C opt test $BUILD_ARGS NO_LFS=1; fi
 
 RUN [ "cross-build-end" ]
 
@@ -45,7 +59,14 @@ FROM redisfab/redis-xbuild:${REDIS_VER}-${ARCH}-${OSNICK}
 
 RUN [ "cross-build-start" ]
 
-RUN set -e; apt-get -qq update; apt-get -q install -y libgomp1
+ARG OSNICK
+ARG OS
+ARG ARCH
+ARG REDIS_VER
+ARG PACK
+
+RUN if [ ! -z $(command -v apt-get) ]; then apt-get -qq update; apt-get -q install -y libgomp1; fi
+RUN if [ ! -z $(command -v yum) ]; then yum install -y libgomp; fi 
 
 ENV REDIS_MODULES /usr/lib/redis/modules
 ENV LD_LIBRARY_PATH $REDIS_MODULES

Dockerfile.arm

@@ -8,11 +8,25 @@ ARG OSNICK=bionic
 # ARCH=x64|arm64v8|arm32v7
 ARG ARCH=x64
 
+# OS=ubuntu18.04|ubuntu16.04|centos7
+ARG OS=ubuntu18.04
+
+ARG CUDA_VER=10.1-cudnn7
+
+ARG PACK=0
+ARG TEST=0
+
 #----------------------------------------------------------------------------------------------
 FROM redisfab/redis:${REDIS_VER}-${ARCH}-${OSNICK} AS redis
-FROM nvidia/cuda:10.1-cudnn7-devel-ubuntu18.04 AS builder
+FROM nvidia/cuda:${CUDA_VER}-devel-${OS} AS builder
+
+ARG OSNICK
+ARG OS
+ARG REDIS_VER
+ARG CUDA_VER
+
+RUN echo "Building for ${OSNICK} (${OS}) [with Redis ${REDIS_VER}]"
 
-# ENV DEPS "git ca-certificates wget unzip cmake libgomp1 patchelf coreutils tcl libjemalloc-dev"
 ENV NVIDIA_VISIBLE_DEVICES all
 ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
 
@@ -31,24 +45,34 @@ RUN if [ "$DEPS_ARGS" = "" ]; then ./get_deps.sh gpu; else env $DEPS_ARGS ./get_
 
 ARG BUILD_ARGS=""
 ADD ./ /build
-RUN make -C opt build GPU=1 $BUILD_ARGS SHOW=1
+RUN set -e ;\
+    . ./opt/readies/bin/sourced ./profile.d ;\
+    make -C opt build GPU=1 $BUILD_ARGS SHOW=1
 
-ARG PACK=0
-ARG TEST=0
+ARG PACK
+ARG TEST
 
+RUN mkdir -p bin/artifacts
 RUN if [ "$PACK" = "1" ]; then make -C opt pack GPU=1; fi
-RUN if [ "$TEST" = "1" ]; then TEST="" make -C opt test GPU=1 $BUILD_ARGS NO_LFS=1; fi
+
+RUN if [ "$TEST" = "1" ]; then TEST= make -C opt test GPU=1 $BUILD_ARGS NO_LFS=1; fi
 
 #----------------------------------------------------------------------------------------------
-FROM nvidia/cuda:10.1-cudnn7-runtime-ubuntu18.04
+FROM nvidia/cuda:${CUDA_VER}-runtime-${OS}
 
-RUN set -e; apt-get -qq update; apt-get -q install -y libgomp1
+ARG OS
+
+RUN if [ ! -z $(command -v apt-get) ]; then apt-get -qq update; apt-get -q install -y libgomp1; fi
+RUN if [ ! -z $(command -v yum) ]; then yum install -y libgomp; fi 
 
 ENV REDIS_MODULES /usr/lib/redis/modules
 RUN mkdir -p $REDIS_MODULES/
 
 COPY --from=redis /usr/local/ /usr/local/
 COPY --from=builder /build/install-gpu/ $REDIS_MODULES/
+
+COPY --from=builder /build/bin/artifacts/ /var/opt/redislabs/artifacts
+
 # COPY --from=builder /usr/local/etc/redis/redis.conf /usr/local/etc/redis/
 
 WORKDIR /data

Dockerfile.gpu

@@ -8,9 +8,15 @@ ARG OSNICK=bionic
 # ARCH=x64|arm64v8|arm32v7
 ARG ARCH=x64
 
+# OS=ubuntu18.04|ubuntu16.04|centos7
+ARG OS=ubuntu18.04
+
+ARG CUDA_VER=10.1-cudnn7
+
+ARG PACK=1
 #----------------------------------------------------------------------------------------------
 FROM redisfab/redis:${REDIS_VER}-${ARCH}-${OSNICK} AS redis
-FROM nvidia/cuda:10.1-cudnn7-devel-ubuntu18.04 AS builder
+FROM nvidia/cuda:${CUDA_VER}-devel-${OS} AS builder
 
 SHELL ["/bin/bash", "-c"]
 
@@ -43,10 +49,11 @@ RUN set -e ;\
 ARG BUILD_ARGS=""
 ADD ./ /build
 RUN set -e ;\
-	. venv/bin/activate ;\
-	make -C opt build GPU=1 $BUILD_ARGS SHOW=1
+    . venv/bin/activate ;\
+    . ./opt/readies/bin/sourced ./profile.d ;\
+    make -C opt build GPU=1 $BUILD_ARGS SHOW=1
 
-ARG PACK=1
+ARG PACK
 
 RUN set -e ;\
 	if [[ $PACK == 1 ]]; then \

Dockerfile.gpu-test

@@ -66,6 +66,8 @@ LIBTORCH=libtorch
 MKL=mkl
 ONNXRUNTIME=onnxruntime
 
+git submodule update --init --recursive || true
+
 ######################################################################################## DLPACK
 
 if [[ $WITH_DLPACK != 0 ]]; then

get_deps.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+# Docker hub does a recursive clone, then checks the branch out,
+# so when a PR adds a submodule (or updates it), it fails.
+git submodule update --init --recursive
+

hooks/post_checkout

@@ -52,5 +52,4 @@ plugins:
   - search
   - include:
       src_path: docs/snippets
-  - versions-menu
-
+  - versions-menu:

mkdocs.yml

@@ -1,5 +1,6 @@
 
 ROOT:=..
+MK.pyver:=3
 include readies/mk/main
 
 MK.cmake:=1
@@ -46,7 +47,6 @@ make cov-upload   # upload coverage data to codecov.io (requires CODECOV_TOKEN)
 
 make pack             # create installation packages
   PACK_DEPS=0           # do not pack dependencies
-  INTO=dir              # place artifacts in `dir`
   BRANCH=name           # use `name` as branch name
 make deploy           # copy packages to S3
 make release          # release a version
@@ -177,10 +177,11 @@ fetch deps:
 
 pack: $(INSTALLED_TARGET)
 	$(SHOW)find $(INSTALL_DIR) -name "*.so" -exec chmod +x {} \;
+	$(SHOW)mkdir -p $(ROOT)/bin/artifacts
 ifneq ($(PACK_DEPS),0)
-	$(SHOW)DEVICE=$(DEVICE) BINDIR=$(BINROOT) INSTALL_DIR=$(INSTALL_DIR) BRANCH=$(BRANCH) INTO=$(INTO) DEPS=1 ./pack.sh
+	$(SHOW)DEVICE=$(DEVICE) BINDIR=$(ROOT)/bin/artifacts INSTALL_DIR=$(INSTALL_DIR) BRANCH=$(BRANCH) DEPS=1 ./pack.sh
 else
-	$(SHOW)DEVICE=$(DEVICE) BINDIR=$(BINROOT) INSTALL_DIR=$(INSTALL_DIR) BRANCH=$(BRANCH) INTO=$(INTO) DEPS=0 ./pack.sh
+	$(SHOW)DEVICE=$(DEVICE) BINDIR=$(ROOT)/bin/artifacts INSTALL_DIR=$(INSTALL_DIR) BRANCH=$(BRANCH) DEPS=0 ./pack.sh
 endif
 
 #----------------------------------------------------------------------------------------------