From dfe59b9481ce1a758574f0bcb5b3d967d1fa6030 Mon Sep 17 00:00:00 2001 From: "Chayim I. Kirshen" Date: Wed, 9 Jun 2021 17:34:56 +0300 Subject: [PATCH 1/7] docker edge build restored --- .circleci/config.yml | 112 +++++++++++++++------------------------ README.md | 2 +- deps/readies | 2 +- docker-compose.debug.yml | 10 ---- docker-compose.yml | 8 --- edge/Dockerfile | 68 ------------------------ edge/Makefile | 68 +++++++----------------- edge/dockerfile.tmpl | 39 ++++++++++++++ edge/sudofile | 1 + 9 files changed, 104 insertions(+), 206 deletions(-) delete mode 100644 docker-compose.debug.yml delete mode 100644 docker-compose.yml delete mode 100755 edge/Dockerfile create mode 100755 edge/dockerfile.tmpl create mode 100644 edge/sudofile diff --git a/.circleci/config.yml b/.circleci/config.yml index 24f6a1a..5de1700 100755 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,6 +1,20 @@ version: 2.1 commands: + + checkout-all: + steps: + - checkout + - run: + name: Checkout submodules + command: git submodule update --init --recursive + + setup-build-system: + steps: + - run: + name: system setup + command: ./deps/readies/bin/getpy3 + setup-executor: steps: - run: @@ -12,75 +26,28 @@ commands: - setup_remote_docker: docker_layer_caching: true - platform-build-steps: +jobs: + + edge-build: parameters: - platform: + variant: type: string publish: - type: boolean - default: false - steps: - - setup-executor - - checkout - - run: - name: Setup - command: | - git submodule update --init --recursive - ./deps/readies/bin/getpy3 - - run: - name: Build for platform - command: | - docker login -u redisfab -p $DOCKER_REDISFAB_PWD - make build OSNICK=<> - <<# parameters.publish >> make publish OSNICK=<> <> - no_output_timeout: 40m - - edge-build-steps: - parameters: - platform: type: string + default: "" + docker: + - image: debian:buster steps: - - setup-executor - - checkout - - run: - name: Setup - command: | - git submodule update --init --recursive - ./deps/readies/bin/getpy3 + - setup_remote_docker + - checkout-all + - setup-build-system - run: - name: Build for platform + name: Build edge for variant command: | docker login -u redisfab -p $DOCKER_REDISFAB_PWD - for i in cpu gpu; do - make -C edge build publish OSNICK=<> PUSH_EDGE=1 DOCKER_ORG=redislabs REDISGEARS_DOCKERDEP=edge REDISTIMESERIES_DOCKERDEP=edge REDISAI_DOCKERDEP=edge-$i VERSION=edge VARIANT=x64-$i - done + make -C edge build DOCKER_SUFFIX=">" VERBOSE=1 <> <> no_output_timeout: 40m -jobs: - platform-build: - parameters: - platform: - type: string - publish: - type: boolean - default: false - docker: - - image: debian:buster - steps: - - platform-build-steps: - platform: <> - publish: <> - - edge-build: - parameters: - platform: - type: string - docker: - - image: debian:buster - steps: - - edge-build-steps: - platform: <> - on-any-branch: &on-any-branch filters: branches: @@ -120,20 +87,29 @@ workflows: version: 2 default_flow: jobs: - - platform-build: - name: build-bionic - platform: bionic - publish: true + - edge-build: + context: common + matrix: + parameters: + variant: + - "GPU=1" + - "CPU=1" <<: *on-any-branch - edge_dailies: + edge_nightly: triggers: - schedule: - cron: "2 9,18 * * *" + cron: "2 3 * * *" filters: branches: only: master jobs: - - platform-build: - name: build-edge - platform: bionic + - edge-build: + context: common + matrix: + parameters: + variant: + - "GPU=1" + - "CPU=1" + publish: + - "PUBLISH=1" diff --git a/README.md b/README.md index 4bf1c3b..c2760d9 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -[![CircleCI](https://circleci.com/gh/RedisLabs/RedisEdge/tree/master.svg?style=svg)](https://circleci.com/gh/RedisLabs/RedisEdge/tree/master) +[![CircleCI](https://circleci.com/gh/RedisLabsModules/RedisEdge/tree/master.svg?style=svg)](https://circleci.com/gh/RedisLabsModules/RedisEdge/tree/master) [![Docker Cloud Build Status](https://img.shields.io/docker/cloud/build/redislabs/redisedge.svg)](https://hub.docker.com/r/redislabs/redisedge/builds/) # redisedge - a Docker image with select Redis Labs modules for the Edge diff --git a/deps/readies b/deps/readies index 05baec3..6f14530 160000 --- a/deps/readies +++ b/deps/readies @@ -1 +1 @@ -Subproject commit 05baec34fbde293eed1b63b5e43edb4d9259e98e +Subproject commit 6f145308b740b8108f1902ef53519b10dbe59fa9 diff --git a/docker-compose.debug.yml b/docker-compose.debug.yml deleted file mode 100644 index f619e54..0000000 --- a/docker-compose.debug.yml +++ /dev/null @@ -1,10 +0,0 @@ -version: '2.1' - -services: - redisedge: - image: redisedge - build: - context: . - dockerfile: Dockerfile - ports: - - 6379:6379 diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index b252dd6..0000000 --- a/docker-compose.yml +++ /dev/null @@ -1,8 +0,0 @@ -version: '2.1' - -services: - redisedge: - image: redisedge - build: . - ports: - - 6379:6379 diff --git a/edge/Dockerfile b/edge/Dockerfile deleted file mode 100755 index e478e9d..0000000 --- a/edge/Dockerfile +++ /dev/null @@ -1,68 +0,0 @@ -# BUILD redisfab/redisedge:${VERSION}-cpu-${ARCH}-${OSNICK} - -ARG VERSION=0.4.0 - -# OSNICK=stretch|bionic|buster -ARG OSNICK=bionic - -# ARCH=x64|arm64v8|arm32v7 -ARG ARCH=x64 - -ARG REDISAI_VERSION=1.0.2 -ARG REDISTIMESERIES_VERSION=1.4.6 -ARG REDISGEARS_VERSION=1.0.2 -ARG REDIS_VER=6.2.1 -ARG DEVICE=cpu - -ARG DOCKER_ORG=redisfab - -# done to specifically allow for ease of optional override with --build-arg -ARG REDISAI_DOCKERDEP=${REDISAI_VERSION}-${DEVICE}-${ARCH}-${OSNICK} -ARG REDISTIMESERIES_DOCKERDEP=${REDISTIMESERIES_VERSION}-${ARCH}-${OSNICK} -ARG REDISGEARS_DOCKERDEP=${REDISGEARS_VERSION}-${ARCH}-${OSNICK} -ARG REDIS_DOCKERDEP=${REDIS_VER}-${ARCH}-${OSNICK} - -#---------------------------------------------------------------------------------------------- -FROM ${DOCKER_ORG}/redisai:${REDISAI_DOCKERDEP} as ai -FROM ${DOCKER_ORG}/redistimeseries:${REDISTIMESERIES_DOCKERDEP} as timeseries -FROM ${DOCKER_ORG}/redisgears:${REDISGEARS_DOCKERDEP} as gears - -#---------------------------------------------------------------------------------------------- -FROM redisfab/redis:${REDIS_DOCKERDEP} - -ARG OSNICK -ARG ARCH -ARG VERSION -ARG REDISAI_VERSION -ARG REDISTIMESERIES_VERSION -ARG REDISGEARS_VERSION -ARG REDIS_VER -ARG REDISTIMESERIES_DOCKERDEP -ARG REDISGEARS_DOCKERDEP -ARG REDIS_DOCKERDEP -ARG REDISAI_DOCKERDEP - -RUN echo "Building redisedge-${OSNICK}:${VERSION}-${DEVICE}-${ARCH} with:" ;\ - echo " RedisAI=${REDISAI_DOCKERDEP}-${DEVICE}" ;\ - echo " RedisTimeSeries=${REDISTIMESERIES_DOCKERDEP}" ;\ - echo " RedisGears=${REDISGEARS_DOCKERDEP}" ;\ - echo " Redis=${REDIS_DOCKERDEP}" - -RUN set -e; if [ ! -z $(command -v apt-get) ]; then apt-get -qq update; apt-get -q install -y libgomp1 git; fi -RUN set -e; if [ ! -z $(command -v yum) ]; then yum install -y libgomp git; fi - -ENV LIBDIR /usr/lib/redis/modules -ENV LD_LIBRARY_PATH $LIBDIR -WORKDIR /data -RUN mkdir -p ${LIBDIR} - -COPY --from=timeseries --chown=redis:redis ${LIBDIR}/*.so ${LIBDIR}/ -COPY --from=ai --chown=redis:redis ${LIBDIR}/ ${LIBDIR}/ -COPY --from=gears --chown=redis:redis /var/opt/redislabs/lib/modules/redisgears.so ${LIBDIR}/ -COPY --from=gears --chown=redis:redis /var/opt/redislabs/modules/ /var/opt/redislabs/modules/ - -WORKDIR /var/opt/redislabs/modules/rg -RUN ln -s python3 python3_`cat /var/opt/redislabs/artifacts/VERSION` - -ADD redisedge.conf /etc -CMD ["/etc/redisedge.conf"] diff --git a/edge/Makefile b/edge/Makefile index 79c207e..66c1377 100755 --- a/edge/Makefile +++ b/edge/Makefile @@ -1,53 +1,21 @@ - -ROOT=.. -MK.pyver:=3 -include $(ROOT)/deps/readies/mk/main - -# OSNICK=buster|stretch|bionic -OSNICK ?= bionic # buster - -REPO=redisedge - -ifeq ($(JETSON),1) -DOCKER_SUFFIX=.jetson -endif - -DOCKERFILE_STEM=$(ROOT)/edge/Dockerfile$(DOCKER_SUFFIX) - -include ../versions - -DOCKER_BUILD_ARGS += \ - REDISAI_VERSION=$(REDISAI_VERSION) \ - REDISTIMESERIES_VERSION=$(REDISTIMESERIES_VERSION) \ - REDISGEARS_VERSION=$(REDISGEARS_VERSION) - -# override defines for the case -ifdef REDISGEARS_DOCKERDEP -DOCKER_BUILD_ARGS += \ - VERSION=edge \ - REDISGEARS_DOCKERDEP=$(REDISGEARS_DOCKERDEP) +DOCKER_ORG=redislabs +PRODUCT=redisedge +VARIANT=cpu +OSNICK=bionic +DEBIAN_OSNICK=buster + +VERISON ?= edge +VARIANT=cpu +ifeq ($(GPU),1) +VARIANT=gpu endif -ifdef REDISAI_DOCKERDEP -DOCKER_BUILD_ARGS += \ - REDISAI_DOCKERDEP=$(REDISAI_DOCKERDEP) -endif - -ifdef REDISTIMESERIES_DOCKERDEP -DOCKER_BUILD_ARGS += \ - REDISTIMESERIES_DOCKERDEP=$(REDISTIMESERIES_DOCKERDEP) -endif - -ifdef REDIS_DOCKERDEP -DOCKER_BUILD_ARGS += \ - REDIS_DOCKERDEP=$(REDIS_DOCKERDEP) -endif - -ifdef DOCKER_ORG -DOCKER_BUILD_ARGS += \ - DOCKER_ORG=$(DOCKER_ORG) -endif - - +export REDIS_DOCKER_ORG=redislabs +export REDISAI_DOCKERTAG=${VERSION}-${VARIANT}-${OSNICK} +export REDISGEARS_DOCKERTAG=${VERSION} +export REDISTIMESERIES_DOCKERTAG=${VERSION} +export REDIS_DOCKERTAG=latest -include $(MK)/docker.native +ROOT=../ +READIES=${ROOT}/deps/readies +include ${READIES}/mk/docker.rules diff --git a/edge/dockerfile.tmpl b/edge/dockerfile.tmpl new file mode 100755 index 0000000..fa814ee --- /dev/null +++ b/edge/dockerfile.tmpl @@ -0,0 +1,39 @@ +# BUILD {{REDIS_DOCKER_ORG}}/redisedge:{{VERSION}}-cpu-{{REDIS_ARCH}}-{{REDIS_OSNICK}} + +#---------------------------------------------------------------------------------------------- +FROM {{REDIS_DOCKER_ORG}}/redisai:{{REDISAI_DOCKERTAG}} as ai +FROM {{REDIS_DOCKER_ORG}}/redistimeseries:{{REDISTIMESERIES_DOCKERTAG}} as timeseries +FROM {{REDIS_DOCKER_ORG}}/redisgears:{{REDISGEARS_DOCKERTAG}} as gears + +#---------------------------------------------------------------------------------------------- +FROM {{REDIS_DOCKER_ORG}}/redis:{{REDIS_DOCKERTAG}} + +COPY --chown=root:root ./edge/sudofile /etc/sudoers.d/99-redislabs + +RUN echo "Building redisedge-{{REDIS_OSNICK}}:{{VERSION}}-{{DEVICE}}-{{REDIS_ARCH}} with:" ;\ + echo " RedisAI={{REDISAI_DOCKERTAG}}";\ + echo " RedisTimeSeries={{REDISTIMESERIES_DOCKERTAG}}" ;\ + echo " RedisGears={{REDISGEARS_DOCKERTAG}}" ;\ + echo " Redis={{REDIS_DOCKERTAG}}" + +ENV LIBDIR /usr/lib/redis/modules +ENV LD_LIBRARY_PATH $LIBDIR + +RUN set -e; if [ ! -z $(command -v apt-get) ]; then sudo apt-get -qq update; sudo apt-get -q install -y libgomp1 git; fi +RUN set -e; if [ ! -z $(command -v yum) ]; then sudo yum install -y libgomp git; fi + +RUN sudo mkdir -p ${LIBDIR} +RUN sudo chown -R redislabs:redislabs ${LIBDIR} + +COPY --from=timeseries --chown=redislabs:redislabs ${LIBDIR}/*.so ${LIBDIR}/ +COPY --from=ai --chown=redislabs:redislabs ${LIBDIR}/ ${LIBDIR}/ +COPY --from=gears --chown=redislabs:redislabs /var/opt/redislabs/lib/modules/redisgears.so ${LIBDIR}/ +COPY --from=gears --chown=redislabs:redislabs /var/opt/redislabs/modules/ /var/opt/redislabs/modules/ + +WORKDIR /var/opt/redislabs/modules/rg +RUN ln -s python3 python3_`cat /var/opt/redislabs/artifacts/VERSION` + +RUN sudo rm /etc/sudoers.d/99-redislabs + +ADD redisedge.conf /etc +CMD ["/etc/redisedge.conf"] diff --git a/edge/sudofile b/edge/sudofile new file mode 100644 index 0000000..75be111 --- /dev/null +++ b/edge/sudofile @@ -0,0 +1 @@ +redislabs ALL=(root) NOPASSWD: ALL From d7bbe3c75cf6e6d0022efb310797949f28a65c0c Mon Sep 17 00:00:00 2001 From: "Chayim I. Kirshen" Date: Wed, 9 Jun 2021 17:38:35 +0300 Subject: [PATCH 2/7] tag for the docker --- edge/Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/edge/Makefile b/edge/Makefile index 66c1377..d634069 100755 --- a/edge/Makefile +++ b/edge/Makefile @@ -4,7 +4,7 @@ VARIANT=cpu OSNICK=bionic DEBIAN_OSNICK=buster -VERISON ?= edge +VERSION ?= edge VARIANT=cpu ifeq ($(GPU),1) VARIANT=gpu @@ -15,6 +15,7 @@ export REDISAI_DOCKERTAG=${VERSION}-${VARIANT}-${OSNICK} export REDISGEARS_DOCKERTAG=${VERSION} export REDISTIMESERIES_DOCKERTAG=${VERSION} export REDIS_DOCKERTAG=latest +DEFAULT_TAG=${DOCKER_ORG}/${PRODUCT}:${VERSION}-${VARIANT} ROOT=../ READIES=${ROOT}/deps/readies From f4d043cc749b60e0ae087429d1ad1555b93672fa Mon Sep 17 00:00:00 2001 From: "Chayim I. Kirshen" Date: Wed, 9 Jun 2021 17:40:59 +0300 Subject: [PATCH 3/7] ci image change to base circle --- .circleci/config.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5de1700..1de4bba 100755 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,7 +7,8 @@ commands: - checkout - run: name: Checkout submodules - command: git submodule update --init --recursive + command: | + git submodule update --init --recursive setup-build-system: steps: @@ -36,7 +37,7 @@ jobs: type: string default: "" docker: - - image: debian:buster + - image: cimg/base:2020.01 steps: - setup_remote_docker - checkout-all From 303c0368c2630bea02981c8b8e3817a843f4d77a Mon Sep 17 00:00:00 2001 From: "Chayim I. Kirshen" Date: Thu, 10 Jun 2021 13:23:56 +0300 Subject: [PATCH 4/7] base image change --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1de4bba..6dae0ee 100755 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -37,7 +37,7 @@ jobs: type: string default: "" docker: - - image: cimg/base:2020.01 + - image: cimg/python:3.9 steps: - setup_remote_docker - checkout-all From 790164e3c83db382d6a05c721eb6e0d91438aefb Mon Sep 17 00:00:00 2001 From: "Chayim I. Kirshen" Date: Thu, 10 Jun 2021 13:26:34 +0300 Subject: [PATCH 5/7] paramters --- .circleci/config.yml | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6dae0ee..0dfc097 100755 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -10,23 +10,6 @@ commands: command: | git submodule update --init --recursive - setup-build-system: - steps: - - run: - name: system setup - command: ./deps/readies/bin/getpy3 - - setup-executor: - steps: - - run: - name: Setup executor - command: | - apt-get -qq update - apt-get -q install -y git openssh-client curl ca-certificates make tar gzip - bash <(curl -fsSL https://get.docker.com) - - setup_remote_docker: - docker_layer_caching: true - jobs: edge-build: @@ -41,12 +24,11 @@ jobs: steps: - setup_remote_docker - checkout-all - - setup-build-system - run: name: Build edge for variant command: | docker login -u redisfab -p $DOCKER_REDISFAB_PWD - make -C edge build DOCKER_SUFFIX=">" VERBOSE=1 <> <> + make -C edge build DOCKER_SUFFIX="<>" VERBOSE=1 <> <> no_output_timeout: 40m on-any-branch: &on-any-branch From 5db79d93027c3ab306829063571b2bea21b7d556 Mon Sep 17 00:00:00 2001 From: "Chayim I. Kirshen" Date: Thu, 10 Jun 2021 13:29:12 +0300 Subject: [PATCH 6/7] restoring py3 --- .circleci/config.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0dfc097..2bc490e 100755 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -10,6 +10,12 @@ commands: command: | git submodule update --init --recursive + setup-build-system: + steps: + - run: + name: system setup + command: ./deps/readies/bin/getpy3 + jobs: edge-build: @@ -23,6 +29,7 @@ jobs: - image: cimg/python:3.9 steps: - setup_remote_docker + - setup-build-system - checkout-all - run: name: Build edge for variant From 8ea81634196336d7bba44bd8fec1bc11ec75dac4 Mon Sep 17 00:00:00 2001 From: "Chayim I. Kirshen" Date: Thu, 10 Jun 2021 13:30:26 +0300 Subject: [PATCH 7/7] after clone --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2bc490e..2734275 100755 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -29,8 +29,8 @@ jobs: - image: cimg/python:3.9 steps: - setup_remote_docker - - setup-build-system - checkout-all + - setup-build-system - run: name: Build edge for variant command: |