stt-datacore · nmoschkin · Oct 3, 2023 · Oct 4, 2023 · Oct 4, 2023 · Oct 4, 2023
diff --git a/.gitignore b/.gitignore
@@ -1,3 +1,4 @@
 data
 .env
-conf
+conf
+multidocker/conf
diff --git a/Dockerfile b/Dockerfile
@@ -26,8 +26,23 @@ RUN apt-get update && DEBIAN_FRONTEND="noninteractive" apt-get install -y \
 
 RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10
 
-RUN curl -sL https://deb.nodesource.com/setup_14.x | bash -
-RUN apt-get install -y nodejs
+## begin node
+
+## old node install commands
+
+# RUN curl -sL https://deb.nodesource.com/setup_14.x | bash -
+# RUN apt-get install -y nodejs
+
+## new node install commands
+
+ARG node_version=v18.20.2
+RUN cd /opt \
+ && curl -LO https://nodejs.org/dist/${node_version}/node-${node_version}-linux-x64.tar.xz \
+ && tar xJf node-${node_version}-linux-x64.tar.xz \
+ && rm node-${node_version}-linux-x64.tar.xz
+ENV PATH=/opt/node-${node_version}-linux-x64/bin:${PATH}
+
+## end node
 
 WORKDIR /
 
@@ -49,6 +64,9 @@ COPY dataorig /dataorig/
 # Clone repos
 WORKDIR /
 #RUN git clone https://github.com/stt-datacore/website.git #- run script should do this
+
+# experimental and easy to revert
+
 RUN git clone https://github.com/stt-datacore/cpp-image-analysis.git
 RUN git clone https://github.com/stt-datacore/bot.git
 RUN git clone https://github.com/stt-datacore/asset-server.git
@@ -95,6 +113,7 @@ RUN make -j$(nproc)
 #RUN mv /asset-server/out/* /dataorig/asset-server/
 #RUN rm -rf /asset-server/out
 #RUN ln -s /data/asset-server /asset-server/out
-
+RUN echo Node Version
+RUN node --version
 COPY ./runme.sh /runme.sh
 ENTRYPOINT /runme.sh
diff --git a/multidocker/.env.defaults b/multidocker/.env.defaults
@@ -1,9 +1,36 @@
-NODE_VERSION=16
+NODE_VERSION=18
 ALPINE_VERSION=3.17
+
 WEBSITE_BRANCH=master
 SERVER_BRANCH=master
 ASSET_BRANCH=master
 BOT_BRANCH=master
-BUILD_TYPE=development
+CPP_BRANCH=master
+
 NGINX_CONF=development.conf
+BUILD_TYPE=development
+
+# PM2
+PM2_INFO_URL=
+PM2_ERROR_URL=
+
+# Shared Assets
+# ASSETS_DIR=/app/asset-server/build/out
+# SQLITE_DB=/app/site-server/static/profiles.sqlite
+
+# MongoDB Environment
+MONGO_INITDB_DATABASE=user
+MONGO_INITDB_ROOT_USERNAME=admin
+MONGO_INITDB_ROOT_PASSWORD=secret
 
+DB_CONNECTION_STRING=sqlite:/sqlite/datacore.db
+MONGO_CONN_STRING=mongodb://admin:secret@datacoredb
+MONGO_DB_NAME=datacore
+MONGO_PROFILE_COLLECTION=playerProfiles
+MONGO_TRACKED_VOYAGES_COLLECTION=trackedVoyages
+MONGO_TRACKED_ASSIGNMENTS_COLLECTION=trackedAssignments
+MONGO_TELEMETRY_VOYAGE_COLLECTION=telemetry
+MONGO_FBB_SOLVES_COLLECTION=solves
+MONGO_FBB_TRIALS_COLLECTION=trials
+MONGO_FBB_COLLECTION=bossBattles
+MONGO_DISCORD_USERS_COLLECTION=discordUsers
diff --git a/multidocker/conf-defaults/bot.env b/multidocker/conf-defaults/bot.env
@@ -0,0 +1,18 @@
+
+# Bot
+BOT_TOKEN=
+DATACORE_URL=https://datacore.app/
+ASSETS_URL=https://assets.datacore.app/
+NODE_ENV=production
+
+IMAGE_ANALYSIS_URL=http://DCImageAnalysis:5000
+
+PROFILE_DATA_PATH=/profiles/
+DB_CONNECTION_STRING=sqlite:/sqlite/datacore.db
+LOG_PATH=/data/logs/
+DC_DATA_PATH=/sitebuild/structured
+CONFIG_PATH=/data/bot_config.json
+
+# ImgFlip
+IMGFLIP_PASSWORD=
+IMGFLIP_USERNAME=
diff --git a/multidocker/conf-defaults/cpp-image-analysis.env b/multidocker/conf-defaults/cpp-image-analysis.env
@@ -0,0 +1,3 @@
+CPP_TRAIN_PATH=/traindata/train/
+CPP_DATA_PATH=/traindata/data/
+
diff --git a/multidocker/conf-defaults/rsync.env b/multidocker/conf-defaults/rsync.env
@@ -0,0 +1 @@
+RSYNC_PASSWORD=insecure
diff --git a/multidocker/conf-defaults/site-server.env b/multidocker/conf-defaults/site-server.env
@@ -0,0 +1,7 @@
+# Site Server Environment
+STT_BOT_USERNAME=
+STT_BOT_PASSWORD=
+JWT_SECRET=
+CORS_ORIGIN=https://datacore.app/
+PROFILE_DATA_PATH=/profiles
+LOG_PATH=/data/logs
diff --git a/multidocker/conf-defaults/website.env b/multidocker/conf-defaults/website.env
@@ -0,0 +1,3 @@
+# Website Environment
+GATSBY_DATACORE_URL=/
+GATSBY_ASSETS_URL=/assets/
diff --git a/multidocker/conf.defaults/bot.env b/multidocker/conf.defaults/bot.env
@@ -0,0 +1,22 @@
+
+# Bot
+BOT_TOKEN=
+DATACORE_URL=https://datacore.app/
+ASSETS_URL=https://assets.datacore.app/
+DEFAULT_GUILD=
+
+# Seperate configuration for bot, because certain dev scenarios
+# require that this be different
+NODE_ENV=development
+
+IMAGE_ANALYSIS_URL=http://DCImageAnalysis:5000
+
+PROFILE_DATA_PATH=/profiles/
+DB_CONNECTION_STRING=sqlite:/sqlite/datacore.db
+LOG_PATH=/data/logs/
+DC_DATA_PATH=/sitebuild/structured
+CONFIG_PATH=/data/bot_config.json
+
+# ImgFlip
+IMGFLIP_PASSWORD=
+IMGFLIP_USERNAME=
diff --git a/multidocker/conf.defaults/cpp-image-analysis.env b/multidocker/conf.defaults/cpp-image-analysis.env
@@ -0,0 +1,3 @@
+CPP_TRAIN_PATH=/traindata/train/
+CPP_DATA_PATH=/traindata/data/
+
diff --git a/multidocker/conf.defaults/site-server.env b/multidocker/conf.defaults/site-server.env
@@ -1,5 +1,7 @@
+# Site Server Environment
 STT_BOT_USERNAME=
 STT_BOT_PASSWORD=
 JWT_SECRET=
-DB_CONNECTION_STRING=sqlite://database
-CORS_ORIGIN=https://datacore.app/
+CORS_ORIGIN=https://datacore.app/
+PROFILE_DATA_PATH=/profiles
+LOG_PATH=/data/logs
diff --git a/multidocker/conf.defaults/website.env b/multidocker/conf.defaults/website.env
@@ -1,3 +1,3 @@
+# Website Environment
 GATSBY_DATACORE_URL=/
-#GATSBY_ASSETS_URL=https://assets.datacore.app/
-GATSBY_ASSETS_URL=/assets/
+GATSBY_ASSETS_URL=/assets/
diff --git a/multidocker/contexts/bot/Dockerfile b/multidocker/contexts/bot/Dockerfile
@@ -11,4 +11,4 @@ COPY bot_config.json /data/
 CMD npm run start
 
 FROM development as production
-CMD pm2-runtime --name server start npm -- start
+CMD pm2-runtime --name bot start npm -- start
diff --git a/multidocker/contexts/cpp-image-analysis/Dockerfile b/multidocker/contexts/cpp-image-analysis/Dockerfile
@@ -6,7 +6,14 @@ ENV LANG=C.UTF-8
 WORKDIR /usr/src/dcimageanalysis
 COPY --from=source . .
 
+WORKDIR /traindata
+RUN mkdir train
+RUN cp /usr/src/dcimageanalysis/data . -r
+
+WORKDIR /usr/src/dcimageanalysis
+
 RUN apk add --no-cache --virtual .build \
+  curl \
   build-base \
   pkgconfig \
   cmake \
@@ -50,4 +57,5 @@ WORKDIR /usr/src/dcimageanalysis/build
 CMD ./imserver
 
 FROM development as production
-CMD pm2-runtime start "./imserver --jsonpath=/website/public_web/structured/ --trainpath=/data/train/ --datapath=/cpp-image-analysis/data/ --asseturl=https://assets.datacore.app/" --name "imageAnalysis"
+
+CMD pm2-runtime start "./imserver --jsonpath=/sitebuild/structured/ --trainpath=$CPP_TRAIN_PATH --datapath=$CPP_DATA_PATH --asseturl=$ASSETS_URL" --name "imageAnalysis"
diff --git a/multidocker/contexts/gittower/Dockerfile b/multidocker/contexts/gittower/Dockerfile
@@ -1,7 +1,8 @@
 ARG ALPINE_VERSION
-FROM alpine:${ALPINE_VERSION}
-RUN apk add docker git bash
-RUN echo */30 * * * * ./runjobs > crontab
+# FROM alpine:${ALPINE_VERSION}
+FROM alpine:latest
+RUN apk add docker git bash docker-compose
+RUN echo "*/5 * * * * /app/runjobs" > crontab
 RUN crontab ./crontab
 RUN mkdir /app
 WORKDIR /app

diff --git a/multidocker/contexts/gittower/gittower b/multidocker/contexts/gittower/gittower
@@ -1,3 +1,4 @@
 #!/bin/bash
 ./setup-jobs
+echo gittower waiting ...
 crond -f -l 8
diff --git a/multidocker/contexts/gittower/runjobs b/multidocker/contexts/gittower/runjobs
@@ -13,3 +13,4 @@ do
     git fetch
     git status|grep "is behind" && reload ${dir##*/}
 done
+echo gittower waiting ...
diff --git a/multidocker/contexts/gittower/setup-jobs b/multidocker/contexts/gittower/setup-jobs
@@ -4,4 +4,5 @@ BASE_URL=https://github.com/stt-datacore/
 
 ./watch website ${BASE_URL}website.git ${WEBSITE_BRANCH}
 ./watch site-server ${BASE_URL}site-server.git ${SERVER_BRANCH}
-./watch asset-server ${BASE_URL}asset-server.git ${ASSET_BRANCH}
+./watch asset-server ${BASE_URL}asset-server.git ${ASSET_BRANCH}
+./watch bot ${BASE_URL}bot.git ${BOT_BRANCH}
diff --git a/multidocker/contexts/nginx/local.conf b/multidocker/contexts/nginx/local.conf
@@ -0,0 +1,31 @@
+server {
+    listen 80;
+    client_max_body_size 100M;
+    #server_name datacore.local;
+
+    location / {
+        root /sitebuild;
+        expires 1h;
+    }
+
+    location /assets/ {
+        root /assets/;
+        access_log off;
+        expires max;
+    }
+
+    location /assets/data/ {
+        root /assets/data/;
+        expires 1h;
+    }
+
+    location /profiles/ {
+        root /profiles/;
+        expires 1h;
+    }
+
+    location /api/ {
+        proxy_pass http://DCSiteServer:4420;
+        expires -1;
+    }
+}
diff --git a/multidocker/contexts/pm2-discord/Dockerfile b/multidocker/contexts/pm2-discord/Dockerfile
@@ -1,7 +1,5 @@
 ARG ALPINE_VERSION
 ARG NODE_VERSION
-ARG PM2_ERROR_URL
-ARG PM2_INFO_URL
 FROM node:${NODE_VERSION}-alpine${ALPINE_VERSION}
 
 RUN npm install pm2 -g
@@ -20,5 +18,7 @@ RUN pm2 set pm2-discord-webhook:stop true
 RUN pm2 set pm2-discord-webhook:restart true
 
 # Set destinations for webhooks (URLs sourced from env)
-RUN pm2 set pm2-discord-webhook:webhook_url_logs ${PM2_INFO_URL}
-RUN pm2 set pm2-discord-webhook:webhook_url_errors ${PM2_ERROR_URL}
+ARG PM2_INFO_URL
+RUN pm2 set pm2-discord-webhook:webhook_url_logs $PM2_INFO_URL
+ARG PM2_ERROR_URL
+RUN pm2 set pm2-discord-webhook:webhook_url_errors $PM2_ERROR_URL
diff --git a/multidocker/contexts/site-server/Dockerfile b/multidocker/contexts/site-server/Dockerfile
@@ -2,7 +2,7 @@ FROM datacore/pm2-discord as development
 RUN apk add python3 build-base sqlite
 RUN mkdir /app
 WORKDIR /app
-COPY --from=source package.json package-lock.json .
+COPY --from=source package.json package-lock.json ./
 RUN npm install
 COPY --from=source . .
 RUN npm run build

diff --git a/multidocker/contexts/website/Dockerfile b/multidocker/contexts/website/Dockerfile
@@ -1,12 +1,14 @@
 ARG NODE_VERSION
 ARG ALPINE_VERSION
-FROM node:${NODE_VERSION}-alpine${ALPINE_VERSION}
+FROM node:$NODE_VERSION-alpine$ALPINE_VERSION
 RUN apk add build-base python3 git rsync
 WORKDIR /app
 # Sentry requires the github sha
 RUN git clone https://github.com/stt-datacore/website.git .git --bare --depth 1
 COPY --from=source package.json yarn.lock ./
+RUN yarn set version 3.6.3
 RUN yarn install
 RUN npx update-browserslist-db@latest 
 COPY --from=source . .
-CMD yarn run build && rsync -ru --delete-after /app/public/ /sitebuild/ && sleep infinity
+
+CMD yarn run build && rsync -ru --delete-after /app/public/ /sitebuild/
diff --git a/multidocker/datacore.db b/multidocker/datacore.db
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,3 @@
		CPP_TRAIN_PATH=/traindata/train/
		CPP_DATA_PATH=/traindata/data/