Skip to content

Commit 1267bf1

Browse files
DvirDukhanDvirDukhan
authored
Merge pull request #585 from RedisAI/v1.2.1
V1.2.1
2 parents eb3a970 + 5e2b734 commit 1267bf1

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

71 files changed

+2616
-1257
lines changed

‎.circleci/config.yml

+84-52
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ commands:
1717
- run:
1818
name: Setup automation
1919
command: |
20-
PIP=1 ./opt/readies/bin/getpy3
20+
./opt/readies/bin/getpy3
2121
./opt/system-setup.py
2222
2323
build-steps:
@@ -70,13 +70,18 @@ commands:
7070
- store_artifacts:
7171
path: tests/logs
7272

73-
platform-build-steps:
74-
parameters:
75-
platform:
76-
type: string
73+
platforms-build-steps:
7774
steps:
78-
- setup-executor
7975
- checkout
76+
- run:
77+
name: Relocate docker overlay2 dir
78+
command: |
79+
sudo systemctl stop docker
80+
sudo mkdir -p /var2/lib/docker
81+
sudo mv /var/lib/docker/overlay2 /var2/lib/docker
82+
sudo mkdir /var/lib/docker/overlay2
83+
sudo mount --bind /var2/lib/docker/overlay2 /var/lib/docker/overlay2
84+
sudo systemctl start docker
8085
- run:
8186
name: Setup
8287
command: |
@@ -87,15 +92,16 @@ commands:
8792
command: |
8893
docker login -u redisfab -p $DOCKER_REDISFAB_PWD
8994
pushd opt/build/docker
90-
#@@ make build publish $(./version-params) CPU=1 OSNICK=<<parameters.platform>> X64=1 ARTIFACTS=1 TEST=1 VERBOSE=1
91-
make build publish $(./version-params) CPU=1 OSNICK=<<parameters.platform>> X64=1 ARTIFACTS=1 VERBOSE=1
92-
make build publish $(./version-params) GPU=1 OSNICK=<<parameters.platform>> X64=1 ARTIFACTS=1 VERBOSE=1
95+
for osnick in bionic xenial; do
96+
make CPU=1 OSNICK=$osnick ARTIFACTS=1 VERBOSE=1
97+
make GPU=1 OSNICK=$osnick ARTIFACTS=1 VERBOSE=1
98+
done
9399
popd > /dev/null
94100
logstar=bin/artifacts/tests-logs-cpu.tgz
95101
logsdir=tests/logs/cpu
96102
mkdir -p $logsdir
97103
if [[ -e $logstar ]]; then tar -C $logsdir -xzf $logstar; fi
98-
(cd bin/artifacts; tar -cf snapshots-<<parameters.platform>>.tar snapshots/)
104+
(cd bin/artifacts; tar -cf snapshots.tar snapshots/)
99105
no_output_timeout: 40m
100106
- persist_to_workspace:
101107
root: bin/
@@ -114,6 +120,7 @@ commands:
114120
- run:
115121
name: Deploy to S3
116122
command: |
123+
du -ah --apparent-size artifacts/*
117124
aws s3 cp artifacts/ s3://redismodules/$PACKAGE_NAME/ --acl public-read --recursive --exclude "*" --include "*.zip" --include "*.tgz"
118125
119126
jobs:
@@ -125,31 +132,31 @@ jobs:
125132
- run:
126133
name: Submodule checkout
127134
command: git submodule update --init --recursive
135+
- setup-automation
128136
- run:
129137
name: lint
130138
command: |
131139
make -C opt lint
132140
133-
build-debian:
141+
build-and-test:
134142
docker:
135-
- image: redisfab/rmbuilder:6.0.5-x64-buster
143+
- image: redisfab/rmbuilder:6.0.9-x64-buster
136144
steps:
137145
- build-steps:
138146
platform: debian
139147

140-
platform-build:
141-
parameters:
142-
platform:
143-
type: string
144-
docker:
145-
- image: debian:buster
148+
platforms-build:
149+
machine:
150+
enabled: true
151+
docker_layer_caching: true
152+
resource_class: gpu.nvidia.small
153+
image: ubuntu-1604-cuda-11.1:202012-01
146154
steps:
147-
- platform-build-steps:
148-
platform: <<parameters.platform>>
155+
- platforms-build-steps
149156

150157
coverage:
151158
docker:
152-
- image: redisfab/rmbuilder:6.0.5-x64-buster
159+
- image: redisfab/rmbuilder:6.0.9-x64-buster
153160
steps:
154161
- checkout
155162
- run:
@@ -177,8 +184,12 @@ jobs:
177184
no_output_timeout: 30m
178185

179186
valgrind:
187+
parameters:
188+
test_args:
189+
type: string
190+
default: "CLUSTER=0 AOF=0"
180191
docker:
181-
- image: redisfab/rmbuilder:6.0.5-x64-buster
192+
- image: redisfab/rmbuilder:6.0.9-x64-buster
182193
steps:
183194
- checkout
184195
- run:
@@ -201,7 +212,7 @@ jobs:
201212
- run:
202213
name: Test with valgrind
203214
command: |
204-
make -C opt test VALGRIND=1
215+
make -C opt test VALGRIND=1 <<parameters.test_args>>
205216
no_output_timeout: 120m
206217

207218
build-macos:
@@ -249,12 +260,21 @@ jobs:
249260
enabled: true
250261
docker_layer_caching: true
251262
resource_class: gpu.nvidia.small
252-
image: ubuntu-1604-cuda-10.1:201909-23
263+
image: ubuntu-1604-cuda-11.1:202012-01
253264
steps:
254265
- checkout
255266
- run:
256267
name: Submodule checkout
257268
command: git submodule update --init --recursive
269+
- run:
270+
name: Relocate docker overlay2 dir
271+
command: |
272+
sudo systemctl stop docker
273+
sudo mkdir -p /var2/lib/docker
274+
sudo mv /var/lib/docker/overlay2 /var2/lib/docker
275+
sudo mkdir /var/lib/docker/overlay2
276+
sudo mount --bind /var2/lib/docker/overlay2 /var/lib/docker/overlay2
277+
sudo systemctl start docker
258278
- run:
259279
name: Build
260280
command: |
@@ -266,53 +286,58 @@ jobs:
266286
docker run --gpus all -v $HOME/tests:/build/tests/logs -it --rm redisai-gpu:latest-x64-bionic-test
267287
no_output_timeout: 40m
268288
- store_artifacts:
269-
path: test/log
289+
path: tests/logs
270290

271291
deploy-artifacts:
272292
parameters:
273293
location:
274294
type: string
275295
docker:
276-
- image: redisfab/rmbuilder:6.0.5-x64-buster
296+
- image: redisfab/rmbuilder:6.0.9-x64-buster
277297
steps:
278298
- attach_workspace:
279299
at: workspace
280300
- run:
281301
name: Deploy to S3
282302
command: |
283303
cd "workspace/artifacts/<<parameters.location>>"
304+
du -ah --apparent-size *
284305
for f in *.zip snapshot/*.tgz; do
285306
aws s3 cp $f s3://redismodules/$PACKAGE_NAME/<<parameters.location>>/ --acl public-read
286307
done
287308
288309
deploy-snapshot:
289310
docker:
290-
- image: redisfab/rmbuilder:6.0.5-x64-buster
311+
- image: redisfab/rmbuilder:6.0.9-x64-buster
291312
steps:
292313
- attach_workspace:
293314
at: workspace
294315
- run:
295316
name: Deploy Snapshots to S3
296317
command: |
297318
cd workspace/artifacts
298-
for f in snapshots-*.tar; do
319+
for f in snapshots*.tar; do
320+
echo "Extracting $f ..."
299321
tar xf $f
300322
done
323+
echo "... done."
324+
du -ah --apparent-size *
301325
cd snapshots
302326
for f in *.zip *.tgz; do
303327
aws s3 cp --no-progress $f s3://redismodules/$PACKAGE_NAME/snapshots/ --acl public-read
304328
done
305329
306330
deploy-release:
307331
docker:
308-
- image: redisfab/rmbuilder:6.0.5-x64-buster
332+
- image: redisfab/rmbuilder:6.0.9-x64-buster
309333
steps:
310334
- attach_workspace:
311335
at: workspace
312336
- run:
313337
name: Deploy Releases to S3
314338
command: |
315339
cd workspace/artifacts
340+
du -ah --apparent-size *
316341
for f in *.zip *.tgz; do
317342
aws s3 cp --no-progress $f s3://redismodules/$PACKAGE_NAME/ --acl public-read
318343
done
@@ -365,48 +390,52 @@ on-integ-and-version-tags: &on-integ-and-version-tags
365390
only: /^v[0-9].*/
366391

367392

368-
platform-build-defs: &platform-build-defs
393+
after-linter: &after-linter
369394
requires:
370-
- build-debian
371-
<<: *on-integ-and-version-tags
372-
# <<: *on-any-branch # debugging
395+
- lint
373396

374-
after-platform-builds: &after-platform-builds
397+
after-build-and-test: &after-build-and-test
375398
requires:
376-
# - build-centos7
377-
- build-bionic
378-
- build-xenial
399+
- build-and-test
379400

380-
after-linter: &after-linter
401+
after-platform-builds: &after-platform-builds
381402
requires:
382-
- lint
403+
- platforms-build
404+
405+
platform-build-defs: &platform-build-defs
406+
<<: *after-build-and-test
407+
<<: *on-integ-and-version-tags
408+
# <<: *on-any-branch # debugging
409+
383410

384411
workflows:
385412
version: 2
386413
build_and_package:
387414
jobs:
388415
- lint:
389416
<<: *on-any-branch
390-
- build-debian:
417+
- build-and-test:
391418
<<: *on-any-branch
392419
<<: *after-linter
393-
- platform-build:
394-
name: build-centos7
395-
platform: centos7
396-
<<: *never # temporarily disabled
397-
- platform-build:
398-
name: build-bionic
399-
platform: bionic
400-
<<: *platform-build-defs
401-
- platform-build:
402-
name: build-xenial
403-
platform: xenial
420+
- platforms-build:
404421
<<: *platform-build-defs
405422
- coverage:
423+
context: common
406424
<<: *on-any-branch
407425
<<: *after-linter
408426
- valgrind:
409-
<<: *on-master
427+
test_args: CLUSTER=0 AOF=0
428+
<<: *on-any-branch
429+
<<: *after-linter
430+
- valgrind:
431+
name: valgrind-cluster
432+
test_args: GEN=0 AOF=0
433+
<<: *on-integ-branch
434+
<<: *after-linter
435+
- valgrind:
436+
name: valgrind-aof
437+
test_args: GEN=0 CLUSTER=0
438+
<<: *on-integ-branch
410439
<<: *after-linter
411440
- build-and-test-gpu:
412441
<<: *on-any-branch
@@ -415,12 +444,15 @@ workflows:
415444
<<: *never # temporarily disabled
416445
# <<: *on-version-tags
417446
- build-multiarch-docker:
447+
context: common
418448
<<: *never # temporarily disabled
419449
# <<: *on-version-tags
420450
- deploy-snapshot:
451+
context: common
421452
<<: *after-platform-builds
422453
<<: *on-integ-branch
423454
- deploy-release:
455+
context: common
424456
<<: *after-platform-builds
425457
<<: *on-version-tags
426458

‎Dockerfile

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# BUILD redisfab/redisai:${VERSION}-cpu-${ARCH}-${OSNICK}
22

3-
ARG REDIS_VER=6.0.5
3+
ARG REDIS_VER=6.0.9
44

55
# OSNICK=bionic|stretch|buster
66
ARG OSNICK=buster
@@ -31,7 +31,7 @@ COPY --from=redis /usr/local/ /usr/local/
3131
COPY ./opt/ opt/
3232
COPY ./tests/flow/test_requirements.txt tests/flow/
3333

34-
RUN PIP=1 FORCE=1 ./opt/readies/bin/getpy3
34+
RUN FORCE=1 ./opt/readies/bin/getpy3
3535
RUN ./opt/system-setup.py
3636

3737
ARG DEPS_ARGS=""

‎Dockerfile.arm

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# BUILD redisfab/redisai:${VERSION}-cpu-${ARCH}-${OSNICK}
22

3-
ARG REDIS_VER=6.0.5
3+
ARG REDIS_VER=6.0.9
44

55
# OSNICK=bionic|stretch|buster
66
ARG OSNICK=buster

‎Dockerfile.gpu

+15-7
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# BUILD redisfab/redisai:${VERSION}-gpu-${ARCH}-${OSNICK}
22

3-
ARG REDIS_VER=6.0.5
3+
ARG REDIS_VER=6.0.9
44

55
# OSNICK=bionic|centos7|centos6
66
ARG OSNICK=bionic
@@ -11,13 +11,14 @@ ARG OS=ubuntu18.04
1111
# ARCH=x64|arm64v8|arm32v7
1212
ARG ARCH=x64
1313

14-
ARG CUDA_VER=10.1-cudnn7
14+
ARG CUDA_VER=11.0-cudnn8
1515

1616
ARG PACK=0
1717
ARG TEST=0
1818

1919
#----------------------------------------------------------------------------------------------
2020
FROM redisfab/redis:${REDIS_VER}-${ARCH}-${OSNICK} AS redis
21+
FROM nvidia/cuda:10.2-cudnn8-devel-${OS} AS cuda_10.2
2122
FROM nvidia/cuda:${CUDA_VER}-devel-${OS} AS builder
2223

2324
ARG OSNICK
@@ -34,10 +35,15 @@ ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
3435
WORKDIR /build
3536
COPY --from=redis /usr/local/ /usr/local/
3637

38+
COPY --from=cuda_10.2 /usr/local/cuda-10.2 /usr/local/cuda-10.2
39+
COPY --from=cuda_10.2 /usr/lib/x86_64-linux-gnu/libcu* /usr/lib/x86_64-linux-gnu/
40+
41+
RUN echo export LD_LIBRARY_PATH=/usr/local/cuda/lib64:/usr/local/cuda-11.0/lib64:/usr/local/cuda-10.2/lib64:$LD_LIBRARY_PATH > /etc/profile.d/cuda.sh
42+
3743
COPY ./opt/ opt/
3844
COPY ./tests/flow/test_requirements.txt tests/flow/
3945

40-
RUN PIP=1 FORCE=1 ./opt/readies/bin/getpy3
46+
RUN FORCE=1 ./opt/readies/bin/getpy3
4147
RUN ./opt/system-setup.py
4248

4349
ARG DEPS_ARGS=""
@@ -59,10 +65,10 @@ RUN set -e ;\
5965

6066
RUN set -e ;\
6167
if [ "$TEST" = "1" ]; then \
62-
TEST= make -C opt test GPU=1 $BUILD_ARGS NO_LFS=1 ;\
63-
if [[ -d test/logs ]]; then \
64-
tar -C test/logs -czf bin/artifacts/test-logs-gpu.tgz . ;\
65-
fi ;\
68+
TEST= make -C opt test GPU=1 $BUILD_ARGS NO_LFS=1 ;\
69+
if [[ -d test/logs ]]; then \
70+
tar -C test/logs -czf bin/artifacts/test-logs-gpu.tgz . ;\
71+
fi ;\
6672
fi
6773

6874
#----------------------------------------------------------------------------------------------
@@ -77,6 +83,8 @@ ENV REDIS_MODULES /usr/lib/redis/modules
7783
RUN mkdir -p $REDIS_MODULES/
7884

7985
COPY --from=redis /usr/local/ /usr/local/
86+
COPY --from=builder /usr/local/cuda-10.2 /usr/local/cuda-10.2
87+
COPY --from=builder /usr/lib/x86_64-linux-gnu/libcu* /usr/lib/x86_64-linux-gnu/
8088
COPY --from=builder /build/install-gpu/ $REDIS_MODULES/
8189

8290
COPY --from=builder /build/bin/artifacts/ /var/opt/redislabs/artifacts

0 commit comments

Comments
 (0)