From a535d0e30bf135f8ac6ab432bf2ddee3502341c8 Mon Sep 17 00:00:00 2001 From: Karun Agarwal <113603846+singhalkarun@users.noreply.github.com> Date: Thu, 1 Aug 2024 12:37:09 +0000 Subject: [PATCH 01/28] migrated services from docker-bhasai, superset to be added as generic service --- bhasai/admin/Caddyfile | 3 + bhasai/admin/docker-compose.yaml | 16 ++++ bhasai/admin/sample.env | 4 + bhasai/ai_tools_proxy/Caddyfile | 3 + bhasai/ai_tools_proxy/docker-compose.yaml | 22 +++++ bhasai/ai_tools_proxy/sample.env | 15 ++++ bhasai/auth_service/Caddyfile | 3 + bhasai/auth_service/docker-compose.yaml | 9 ++ bhasai/auth_service/sample.env | 2 + bhasai/autotune/Caddyfile | 3 + bhasai/autotune/docker-compose.yaml | 49 +++++++++++ bhasai/autotune/sample.env | 27 ++++++ bhasai/bff/Caddyfile | 3 + bhasai/bff/docker-compose.yaml | 48 +++++++++++ bhasai/bff/sample.env | 22 +++++ bhasai/dataset_service/Caddyfile | 3 + bhasai/dataset_service/docker-compose.yaml | 19 +++++ bhasai/dataset_service/sample.env | 9 ++ bhasai/deployer/Caddyfile | 3 + bhasai/deployer/docker-compose.yaml | 20 +++++ bhasai/deployer/sample.env | 10 +++ bhasai/document_service/Caddyfile | 4 + bhasai/document_service/docker-compose.yaml | 22 +++++ bhasai/document_service/sample.env | 7 ++ bhasai/inboundjs/Caddyfile | 3 + bhasai/inboundjs/docker-compose.yaml | 13 +++ bhasai/inboundjs/sample.env | 5 ++ bhasai/nodehub/Caddyfile | 3 + bhasai/nodehub/docker-compose.yaml | 11 +++ bhasai/nodehub/sample.env | 3 + bhasai/orchestrator/docker-compose.yaml | 9 ++ bhasai/orchestrator/sample.env | 2 + bhasai/pdf_parser/Caddyfile | 3 + bhasai/pdf_parser/docker-compose.yaml | 82 +++++++++++++++++++ bhasai/pdf_parser/sample.env | 37 +++++++++ bhasai/telemetry/Caddyfile | 3 + bhasai/telemetry/docker-compose.yaml | 13 +++ bhasai/telemetry/sample.env | 6 ++ bhasai/transport_socket/Caddyfile | 3 + bhasai/transport_socket/docker-compose.yaml | 12 +++ bhasai/transport_socket/sample.env | 2 + bhasai/uci_apis/Caddyfile | 3 + bhasai/uci_apis/docker-compose.yaml | 19 +++++ bhasai/uci_apis/sample.env | 8 ++ bhasai/user_management_service/Caddyfile | 3 + .../docker-compose.yaml | 11 +++ bhasai/user_management_service/sample.env | 4 + bhasai/vault/Dockerfile | 4 + bhasai/vault/admin-policy.hcl | 59 +++++++++++++ bhasai/vault/consul-init.sh | 32 ++++++++ bhasai/vault/docker-compose.yaml | 22 +++++ bhasai/vault/env.tpl | 2 + bhasai/vault/vault-init.sh | 79 ++++++++++++++++++ bhasai/vault/vault.json | 14 ++++ bhasai/yt_parser/Caddyfile | 3 + bhasai/yt_parser/docker-compose.yaml | 51 ++++++++++++ bhasai/yt_parser/sample.env | 19 +++++ 57 files changed, 869 insertions(+) create mode 100644 bhasai/admin/Caddyfile create mode 100644 bhasai/admin/docker-compose.yaml create mode 100644 bhasai/admin/sample.env create mode 100644 bhasai/ai_tools_proxy/Caddyfile create mode 100644 bhasai/ai_tools_proxy/docker-compose.yaml create mode 100644 bhasai/ai_tools_proxy/sample.env create mode 100644 bhasai/auth_service/Caddyfile create mode 100644 bhasai/auth_service/docker-compose.yaml create mode 100644 bhasai/auth_service/sample.env create mode 100644 bhasai/autotune/Caddyfile create mode 100644 bhasai/autotune/docker-compose.yaml create mode 100644 bhasai/autotune/sample.env create mode 100644 bhasai/bff/Caddyfile create mode 100644 bhasai/bff/docker-compose.yaml create mode 100644 bhasai/bff/sample.env create mode 100644 bhasai/dataset_service/Caddyfile create mode 100644 bhasai/dataset_service/docker-compose.yaml create mode 100644 bhasai/dataset_service/sample.env create mode 100644 bhasai/deployer/Caddyfile create mode 100644 bhasai/deployer/docker-compose.yaml create mode 100644 bhasai/deployer/sample.env create mode 100644 bhasai/document_service/Caddyfile create mode 100644 bhasai/document_service/docker-compose.yaml create mode 100644 bhasai/document_service/sample.env create mode 100644 bhasai/inboundjs/Caddyfile create mode 100644 bhasai/inboundjs/docker-compose.yaml create mode 100644 bhasai/inboundjs/sample.env create mode 100644 bhasai/nodehub/Caddyfile create mode 100644 bhasai/nodehub/docker-compose.yaml create mode 100644 bhasai/nodehub/sample.env create mode 100644 bhasai/orchestrator/docker-compose.yaml create mode 100644 bhasai/orchestrator/sample.env create mode 100644 bhasai/pdf_parser/Caddyfile create mode 100644 bhasai/pdf_parser/docker-compose.yaml create mode 100644 bhasai/pdf_parser/sample.env create mode 100644 bhasai/telemetry/Caddyfile create mode 100644 bhasai/telemetry/docker-compose.yaml create mode 100644 bhasai/telemetry/sample.env create mode 100644 bhasai/transport_socket/Caddyfile create mode 100644 bhasai/transport_socket/docker-compose.yaml create mode 100644 bhasai/transport_socket/sample.env create mode 100644 bhasai/uci_apis/Caddyfile create mode 100644 bhasai/uci_apis/docker-compose.yaml create mode 100644 bhasai/uci_apis/sample.env create mode 100644 bhasai/user_management_service/Caddyfile create mode 100644 bhasai/user_management_service/docker-compose.yaml create mode 100644 bhasai/user_management_service/sample.env create mode 100644 bhasai/vault/Dockerfile create mode 100644 bhasai/vault/admin-policy.hcl create mode 100755 bhasai/vault/consul-init.sh create mode 100644 bhasai/vault/docker-compose.yaml create mode 100644 bhasai/vault/env.tpl create mode 100644 bhasai/vault/vault-init.sh create mode 100644 bhasai/vault/vault.json create mode 100644 bhasai/yt_parser/Caddyfile create mode 100644 bhasai/yt_parser/docker-compose.yaml create mode 100644 bhasai/yt_parser/sample.env diff --git a/bhasai/admin/Caddyfile b/bhasai/admin/Caddyfile new file mode 100644 index 0000000..9b962a1 --- /dev/null +++ b/bhasai/admin/Caddyfile @@ -0,0 +1,3 @@ +{$DOMAIN_NAME}://admin.{$DOMAIN_NAME} { + reverse_proxy admin:3000 +} \ No newline at end of file diff --git a/bhasai/admin/docker-compose.yaml b/bhasai/admin/docker-compose.yaml new file mode 100644 index 0000000..0af09fd --- /dev/null +++ b/bhasai/admin/docker-compose.yaml @@ -0,0 +1,16 @@ +services: + admin: + build: + context: https://${GIT_USERNAME}:${GIT_PASSWORD}@github.com/${org}/admin.git#${ADMIN_GITHUB_BRANCH:-${DEFAULT_GITHUB_BRANCH:?DEFAULT_GITHUB_BRANCH is not set}} + args: + NEXT_PUBLIC_UCI_API_URL: ${DOMAIN_SCHEME:-https}://uci-apis.${DOMAIN_NAME} + NEXT_PUBLIC_BFF_API_URL: ${DOMAIN_SCHEME:-https}://bff.${DOMAIN_NAME} + NEXT_PUBLIC_SOCKET_URL: ${DOMAIN_SCHEME:-https}://ts.${DOMAIN_NAME} + NEXT_PUBLIC_AUTH_SERVICE_URL: ${DOMAIN_SCHEME:-https}://auth-service.${DOMAIN_NAME} + NEXT_PUBLIC_DATASET_SERVICE_URL: ${DOMAIN_SCHEME:-https}://dataset-service.${DOMAIN_NAME} + NEXT_PUBLIC_DOC_SERVICE_URL: ${DOMAIN_SCHEME:-https}://document-service.${DOMAIN_NAME} + NEXT_PUBLIC_PWA_DEPLOYER_URL: ${DOMAIN_SCHEME:-https}://deployer.${DOMAIN_NAME} + NEXT_PUBLIC_TELEMETRY_API_URL: ${DOMAIN_SCHEME:-https}://telemetry.${DOMAIN_NAME} + NEXT_PUBLIC_FLOWISE_URL: ${NEXT_PUBLIC_FLOWISE_URL} + NEXT_PUBLIC_AUTOTUNE_API_URL: ${DOMAIN_SCHEME:-https}://autotune.${DOMAIN_NAME} + restart: always diff --git a/bhasai/admin/sample.env b/bhasai/admin/sample.env new file mode 100644 index 0000000..0b9ee98 --- /dev/null +++ b/bhasai/admin/sample.env @@ -0,0 +1,4 @@ +#admin +DOMAIN_SCHEME= +DOMAIN_NAME= +NEXT_PUBLIC_FLOWISE_URL= \ No newline at end of file diff --git a/bhasai/ai_tools_proxy/Caddyfile b/bhasai/ai_tools_proxy/Caddyfile new file mode 100644 index 0000000..644a240 --- /dev/null +++ b/bhasai/ai_tools_proxy/Caddyfile @@ -0,0 +1,3 @@ +{$DOMAIN_SCHEME}://ai-tools-proxy.{$DOMAIN_NAME} { + reverse_proxy ai_tools_proxy:3000 +} diff --git a/bhasai/ai_tools_proxy/docker-compose.yaml b/bhasai/ai_tools_proxy/docker-compose.yaml new file mode 100644 index 0000000..a6cefc7 --- /dev/null +++ b/bhasai/ai_tools_proxy/docker-compose.yaml @@ -0,0 +1,22 @@ + services: + ai_tools_proxy: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/ai-tools-proxy:${AI_TOOLS_PROXY_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + restart: always + mem_limit: ${AI_TOOLS_PROXY_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${AI_TOOLS_PROXY_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + environment: + DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/ai_tools_proxy" + DEFAULT_RETRY_COUNT: ${DEFAULT_RETRY_COUNT} + DEFAULT_TIMEOUT: ${DEFAULT_TIMEOUT} + ULCA_API_KEY: ${ULCA_API_KEY} + ULCA_CONFIG_URL: ${ULCA_CONFIG_URL} + ULCA_USER_ID: ${ULCA_USER_ID} + MINIO_ENDPOINT: ${DOMAIN_SCHEME:-https}://cdn-api.${DOMAIN_NAME} + MINIO_PORT: 443 + MINIO_SECURE_CONN: "True" + MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} + MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} + MINIO_BUCKET: ai-tools-proxy + ENVIRONMENT: ${env} + TELEMETRY_URL: http://telemetry:3000 + OPENAI_API_KEY: ${OPENAI_API_KEY} diff --git a/bhasai/ai_tools_proxy/sample.env b/bhasai/ai_tools_proxy/sample.env new file mode 100644 index 0000000..c59d1a5 --- /dev/null +++ b/bhasai/ai_tools_proxy/sample.env @@ -0,0 +1,15 @@ +#ai_tools_proxy +POSTGRES_USER= +POSTGRES_PASSWORD= +DEFAULT_RETRY_COUNT= +DEFAULT_TIMEOUT= +ULCA_API_KEY= +ULCA_CONFIG_URL= +ULCA_USER_ID= +DOMAIN_SCHEME= +DOMAIN_NAME= +MINIO_ACCESS_KEY= +MINIO_SECRET_KEY= +env= +OPENAI_API_KEY= + diff --git a/bhasai/auth_service/Caddyfile b/bhasai/auth_service/Caddyfile new file mode 100644 index 0000000..f02ce04 --- /dev/null +++ b/bhasai/auth_service/Caddyfile @@ -0,0 +1,3 @@ +{$DOMAIN_SCHEME}://auth-service.{$DOMAIN_NAME} { + reverse_proxy auth_service:3000 +} \ No newline at end of file diff --git a/bhasai/auth_service/docker-compose.yaml b/bhasai/auth_service/docker-compose.yaml new file mode 100644 index 0000000..f52215b --- /dev/null +++ b/bhasai/auth_service/docker-compose.yaml @@ -0,0 +1,9 @@ +services: + auth_service: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/auth-service:${AUTH_SERVICE_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + restart: always + mem_limit: ${AUTH_SERVICE_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${AUTH_SERVICE_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + environment: + FUSIONAUTH_URL: http://fusionauth:9011 + FUSIONAUTH_KEY: ${FUSIONAUTH_API_KEY} \ No newline at end of file diff --git a/bhasai/auth_service/sample.env b/bhasai/auth_service/sample.env new file mode 100644 index 0000000..04cf495 --- /dev/null +++ b/bhasai/auth_service/sample.env @@ -0,0 +1,2 @@ +#auth_service +FUSIONAUTH_API_KEY= \ No newline at end of file diff --git a/bhasai/autotune/Caddyfile b/bhasai/autotune/Caddyfile new file mode 100644 index 0000000..280f9dd --- /dev/null +++ b/bhasai/autotune/Caddyfile @@ -0,0 +1,3 @@ +{$DOMAIN_SCHEME}://autotune.{$DOMAIN_NAME} { + reverse_proxy autotune:8000 +} diff --git a/bhasai/autotune/docker-compose.yaml b/bhasai/autotune/docker-compose.yaml new file mode 100644 index 0000000..00f470f --- /dev/null +++ b/bhasai/autotune/docker-compose.yaml @@ -0,0 +1,49 @@ + services: + autotune: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/autotune:${AUTOTUNE_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + restart: always + mem_limit: ${AUTOTUNE_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${AUTOTUNE_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + command: bash -c "poetry install && python manage.py database && python manage.py runserver 0.0.0.0:8000" + environment: + DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/autotune" + REDIS_URL: redis://redis:6379/7 + OPENAI_API_KEY: ${OPENAI_API_KEY} + MINIO_BASE_URL: "${DOMAIN_SCHEME:-https}://cdn-api.${DOMAIN_NAME}" + MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} + MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} + MINIO_BUCKET: autotune + MINIO_SECURE_CONN: "True" + CELERY_MAX_RETRIES: ${CELERY_MAX_RETRIES} + DJANGO_SECRET_KEY: ${DJANGO_SECRET_KEY} + HUGGING_FACE_USERNAME: ${HUGGING_FACE_USERNAME} + HUGGING_FACE_TOKEN: ${HUGGING_FACE_TOKEN} + GITHUB_PAT: ${GITHUB_PAT} + + autotune_celery_worker: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/autotune:${AUTOTUNE_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + restart: always + mem_limit: ${AUTOTUNE_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${AUTOTUNE_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + command: bash -c "poetry install && celery -A autotune worker --loglevel=info -P gevent" + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [gpu] + environment: + DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/autotune" + REDIS_URL: redis://redis:6379/7 + OPENAI_API_KEY: ${OPENAI_API_KEY} + MINIO_BASE_URL: "${DOMAIN_SCHEME:-https}://cdn-api.${DOMAIN_NAME}" + MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} + MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} + MINIO_BUCKET: autotune + MINIO_SECURE_CONN: "True" + CELERY_MAX_RETRIES: ${CELERY_MAX_RETRIES} + DJANGO_SECRET_KEY: ${DJANGO_SECRET_KEY} + HUGGING_FACE_USERNAME: ${HUGGING_FACE_USERNAME} + HUGGING_FACE_TOKEN: ${HUGGING_FACE_TOKEN} + GITHUB_PAT: ${GITHUB_PAT} diff --git a/bhasai/autotune/sample.env b/bhasai/autotune/sample.env new file mode 100644 index 0000000..dfff2ae --- /dev/null +++ b/bhasai/autotune/sample.env @@ -0,0 +1,27 @@ +#autotune +POSTGRES_USER= +POSTGRES_PASSWORD= +OPENAI_API_KEY= +DOMAIN_NAME= +DOMAIN_SCHEME= +MINIO_ACCESS_KEY= +MINIO_SECRET_KEY= +CELERY_MAX_RETRIES= +DJANGO_SECRET_KEY= +HUGGING_FACE_USERNAME= +HUGGING_FACE_TOKEN= +GITHUB_PAT= + +#autotune_celery_worker +POSTGRES_USER= +POSTGRES_PASSWORD= +OPENAI_API_KEY= +DOMAIN_NAME= +DOMAIN_SCHEME= +MINIO_ACCESS_KEY= +MINIO_SECRET_KEY= +CELERY_MAX_RETRIES= +DJANGO_SECRET_KEY= +HUGGING_FACE_USERNAME= +HUGGING_FACE_TOKEN= +GITHUB_PAT= \ No newline at end of file diff --git a/bhasai/bff/Caddyfile b/bhasai/bff/Caddyfile new file mode 100644 index 0000000..999bb55 --- /dev/null +++ b/bhasai/bff/Caddyfile @@ -0,0 +1,3 @@ +{$DOMAIN_SCHEME}://bff.{$DOMAIN_NAME} { + reverse_proxy bff:3000 +} \ No newline at end of file diff --git a/bhasai/bff/docker-compose.yaml b/bhasai/bff/docker-compose.yaml new file mode 100644 index 0000000..77a76f8 --- /dev/null +++ b/bhasai/bff/docker-compose.yaml @@ -0,0 +1,48 @@ +services: + bff: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/bff:${BFF_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + restart: always + mem_limit: ${BFF_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${BFF_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + depends_on: + db: + condition: service_healthy + redis: + condition: service_started + environment: + BFF_BASE_URL: http://bff:3000 + EXCEL_PARSER_URL: http://excel_parser:8000 + DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/bff?sslmode=disable&connect_timeout=100" + JWKS_URI: ${APP_JWKS_URI} + MINIO_PDF_BUCKET: bff + MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} + MINIO_ENDPOINT: ${DOMAIN_SCHEME:-https}://cdn-api.${DOMAIN_NAME} + MINIO_PORT: 443 + MINIO_SECURE_CONN: "True" + MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} + OPENAI_API_KEY: ${OPENAI_API_KEY} + PDF_PARSER_BASE_URL: ${DOMAIN_SCHEME:-https}://pdf-parser.${DOMAIN_NAME} + PDF_SPLIT: '50' + REDIS_DB: 8 + REDIS_HOST: redis + REDIS_PORT: 6379 + USER_SERVICE_BASE_URL: http://user_service:3000 + OUTBOUND_BASE_URL: http://inboundjs:3000/outbound + TRANSPORT_SOCKET_BASE_URL: http://transport_socket:3000 + UCI_APIS_BASE_URL: http://uci_apis:3000 + ULCA_API_KEY: ${ULCA_API_KEY} + ULCA_CONFIG_URL: ${ULCA_CONFIG_URL} + ULCA_USER_ID: ${ULCA_USER_ID} + YT_PARSER_URL: http://yt_parser:8000 + CHAT_WITH_DOCS_ORG_OWNER_EMAIL: ${CHAT_WITH_DOCS_ORG_OWNER_EMAIL} + CHAT_WITH_DOCS_ORG_OWNER_PASSWORD: ${CHAT_WITH_DOCS_ORG_OWNER_PASSWORD} + CHAT_WITH_DOCS_PASSWORD_KEY: ${CHAT_WITH_DOCS_PASSWORD_KEY} + CHAT_WITH_DOCS_ORG_ID: ${CHAT_WITH_DOCS_ORG_ID} + REGISTRY_URL: ${REGISTRY_URL} + INBOUNDJS_BASE_URL: http://inboundjs:3000 + WEATHER_VISUAL_CROSSING_BASE_URL: ${WEATHER_VISUAL_CROSSING_BASE_URL} + WEATHER_VISUAL_CROSSING_API_KEY: ${WEATHER_VISUAL_CROSSING_API_KEY} + DOCUMENT_SERVICE_BASE_URL: ${DOMAIN_SCHEME:-https}://document-service.${DOMAIN_NAME} + AZURE_URL: ${AZURE_URL} + USER_SERVICE_ADMIN_API_KEY: ${USER_SERVICE_ADMIN_API_KEY} + USER_SERVICE_URL: ${USER_SERVICE_URL} diff --git a/bhasai/bff/sample.env b/bhasai/bff/sample.env new file mode 100644 index 0000000..021c66d --- /dev/null +++ b/bhasai/bff/sample.env @@ -0,0 +1,22 @@ +#bff +POSTGRES_USER= +POSTGRES_PASSWORD= +APP_JWKS_URI= +MINIO_ACCESS_KEY= +MINIO_SECRET_KEY= +DOMAIN_NAME= +DOMAIN_SCHEME= +OPENAI_API_KEY= +ULCA_API_KEY= +ULCA_CONFIG_URL= +ULCA_USER_ID= +CHAT_WITH_DOCS_ORG_OWNER_EMAIL= +CHAT_WITH_DOCS_ORG_OWNER_PASSWORD= +CHAT_WITH_DOCS_PASSWORD_KEY= +CHAT_WITH_DOCS_ORG_ID= +REGISTRY_URL= +WEATHER_VISUAL_CROSSING_BASE_URL= +WEATHER_VISUAL_CROSSING_API_KEY= +AZURE_URL= +USER_SERVICE_ADMIN_API_KEY= +USER_SERVICE_URL= diff --git a/bhasai/dataset_service/Caddyfile b/bhasai/dataset_service/Caddyfile new file mode 100644 index 0000000..a80670c --- /dev/null +++ b/bhasai/dataset_service/Caddyfile @@ -0,0 +1,3 @@ +{$DOMAIN_SCHEME}://dataset-service.{$DOMAIN_NAME} { + reverse_proxy dataset_service:3000 +} diff --git a/bhasai/dataset_service/docker-compose.yaml b/bhasai/dataset_service/docker-compose.yaml new file mode 100644 index 0000000..2416049 --- /dev/null +++ b/bhasai/dataset_service/docker-compose.yaml @@ -0,0 +1,19 @@ +services: + dataset_service: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/dataset-service:${DATASET_SERVICE_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + restart: always + mem_limit: ${DATASET_SERVICE_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${DATASET_SERVICE_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + environment: + DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/dataset_service" + MINIO_ENDPOINT: "${DOMAIN_SCHEME}://cdn-api.${DOMAIN_NAME}" + MINIO_SECURE_CONN: "True" + MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} + MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} + MINIO_BUCKET: dataset-service + OPENAI_API_KEY: ${OPENAI_API_KEY} + REDIS_HOST: redis + REDIS_PORT: 6379 + REDIS_DB: 9 + AI_TOOLS_EMBEDDING_BASE_URL: ${DOMAIN_SCHEME:-https}://ai-tools.${DOMAIN_NAME} + AI_TOOLS_BATCH_SIZE: ${AI_TOOLS_EMBEDDING_BATCH_SIZE} diff --git a/bhasai/dataset_service/sample.env b/bhasai/dataset_service/sample.env new file mode 100644 index 0000000..2e0b620 --- /dev/null +++ b/bhasai/dataset_service/sample.env @@ -0,0 +1,9 @@ +#dataset_service +DOMAIN_NAME= +DOMAIN_SCHEME= +MINIO_ACCESS_KEY= +MINIO_SECRET_KEY= +POSTGRES_USER= +POSTGRES_PASSWORD= +AI_TOOLS_EMBEDDING_BATCH_SIZE= +OPENAI_API_KEY= diff --git a/bhasai/deployer/Caddyfile b/bhasai/deployer/Caddyfile new file mode 100644 index 0000000..408c0ee --- /dev/null +++ b/bhasai/deployer/Caddyfile @@ -0,0 +1,3 @@ +{$DOMAIN_NAME}://deployer.{$DOMAIN_NAME} { + reverse_proxy deployer:3000 +} \ No newline at end of file diff --git a/bhasai/deployer/docker-compose.yaml b/bhasai/deployer/docker-compose.yaml new file mode 100644 index 0000000..4e9cada --- /dev/null +++ b/bhasai/deployer/docker-compose.yaml @@ -0,0 +1,20 @@ +services: + deployer: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/deployer:${DEPLOYER_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + restart: always + mem_limit: ${DEPLOYER_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${DEPLOYER_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + environment: + DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/deployer?sslmode=disable&connect_timeout=100" + VERCEL_TOKEN: ${DEPLOYER_VERCEL_TOKEN} + PWA_REPO: ${DEPLOYER_PWA_REPO} + APP_ENV: ${DEPLOYER_APP_ENV} + SELF_BASE_URL: ${DOMAIN_SCHEME:-https}://deployer.${DOMAIN_NAME} + MINIO_BUCKETNAME: deployer + STORAGE_ENDPOINT: ${DOMAIN_SCHEME:-https}://cdn-api.${DOMAIN_NAME} + STORAGE_PORT: 443 + STORAGE_USE_SSL: 'true' + STORAGE_ACCESS_KEY: ${MINIO_ACCESS_KEY} + STORAGE_SECRET_KEY: ${MINIO_SECRET_KEY} + STORAGE_MODE: "minio" + AUTH_SERVICE_URL: http://auth_service:3000 \ No newline at end of file diff --git a/bhasai/deployer/sample.env b/bhasai/deployer/sample.env new file mode 100644 index 0000000..89f65bd --- /dev/null +++ b/bhasai/deployer/sample.env @@ -0,0 +1,10 @@ +#deployer +POSTGRES_USER= +POSTGRES_PASSWORD= +DEPLOYER_VERCEL_TOKEN= +DEPLOYER_PWA_REPO= +DEPLOYER_APP_ENV= +DOMAIN_SCHEME= +DOMAIN_NAME= +MINIO_ACCESS_KEY= +MINIO_SECRET_KEY= \ No newline at end of file diff --git a/bhasai/document_service/Caddyfile b/bhasai/document_service/Caddyfile new file mode 100644 index 0000000..de8c4df --- /dev/null +++ b/bhasai/document_service/Caddyfile @@ -0,0 +1,4 @@ +{$DOMAIN_SCHEME}://document-service.{$DOMAIN_NAME} { + reverse_proxy document_service:3000 +} + diff --git a/bhasai/document_service/docker-compose.yaml b/bhasai/document_service/docker-compose.yaml new file mode 100644 index 0000000..53700ba --- /dev/null +++ b/bhasai/document_service/docker-compose.yaml @@ -0,0 +1,22 @@ +services: + document_service: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/document-service:${DOCUMENT_SERVICE_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + restart: always + mem_limit: ${DOCUMENT_SERVICE_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${DOCUMENT_SERVICE_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + environment: + DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/document_service" + MINIO_ENDPOINT: "minio" + MINIO_PORT: "9000" + MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} + MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} + MINIO_BUCKET: document-service + MINIO_SECURE_CONN: "False" + OPENAI_API_KEY: ${OPENAI_API_KEY} + PDF_PARSER_URL: http://pdf_parser:8000 + YT_PARSER_URL: http://yt_parser:8000 + REDIS_HOST: redis + REDIS_PORT: 6379 + REDIS_DB: 4 + AI_TOOLS_EMBEDDING_BASE_URL: ${DOMAIN_SCHEME:-https}://ai-tools.${DOMAIN_NAME} + AI_TOOLS_BATCH_SIZE: ${AI_TOOLS_EMBEDDING_BATCH_SIZE} diff --git a/bhasai/document_service/sample.env b/bhasai/document_service/sample.env new file mode 100644 index 0000000..40c8da8 --- /dev/null +++ b/bhasai/document_service/sample.env @@ -0,0 +1,7 @@ +#document_service +POSTGRES_USER= +POSTGRES_PASSWORD= +MINIO_ACCESS_KEY= +MINIO_SECRET_KEY= +OPENAI_API_KEY= +AI_TOOLS_EMBEDDING_BATCH_SIZE= \ No newline at end of file diff --git a/bhasai/inboundjs/Caddyfile b/bhasai/inboundjs/Caddyfile new file mode 100644 index 0000000..340d8f9 --- /dev/null +++ b/bhasai/inboundjs/Caddyfile @@ -0,0 +1,3 @@ +{$DOMAIN_NAME}://inboundjs.{$DOMAIN_NAME} { + reverse_proxy inboundjs:3000 +} \ No newline at end of file diff --git a/bhasai/inboundjs/docker-compose.yaml b/bhasai/inboundjs/docker-compose.yaml new file mode 100644 index 0000000..56ab024 --- /dev/null +++ b/bhasai/inboundjs/docker-compose.yaml @@ -0,0 +1,13 @@ +services: + inboundjs: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/inbound-js:${INBOUNDJS_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + mem_limit: ${INBOUNDJS_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${INBOUNDJS_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + restart: always + environment: + DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/inboundjs" + ORCHESTRATOR_API_ENDPOINT: "http://orchestrator:3000" + UCI_API_BASE_URL: http://uci-apis:3000 + UCI_API_TOKEN: ${UCI_ADMIN_TOKEN} + TELEMETRY_SERVICE_HOST: http://telemetry:3000 + ENVIRONMENT: ${env} \ No newline at end of file diff --git a/bhasai/inboundjs/sample.env b/bhasai/inboundjs/sample.env new file mode 100644 index 0000000..3a8f5c4 --- /dev/null +++ b/bhasai/inboundjs/sample.env @@ -0,0 +1,5 @@ +#inboundjs +env= +UCI_ADMIN_TOKEN= +POSTGRES_USER= +POSTGRES_PASSWORD= \ No newline at end of file diff --git a/bhasai/nodehub/Caddyfile b/bhasai/nodehub/Caddyfile new file mode 100644 index 0000000..8bd6763 --- /dev/null +++ b/bhasai/nodehub/Caddyfile @@ -0,0 +1,3 @@ +{$DOMAIN_NAME}://nodehub.{$DOMAIN_NAME} { + reverse_proxy nodehub:3000 +} \ No newline at end of file diff --git a/bhasai/nodehub/docker-compose.yaml b/bhasai/nodehub/docker-compose.yaml new file mode 100644 index 0000000..413697a --- /dev/null +++ b/bhasai/nodehub/docker-compose.yaml @@ -0,0 +1,11 @@ +services: + nodehub: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/nodehub:${NODEHUB_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + mem_limit: ${NODEHUB_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${NODEHUB_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + restart: always + environment: + DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/nodehub" + AUTH_SERVICE_URL: "http://auth_service:3000" + REGISTRY_BASE_URL: ${REGISTRY_BASE_URL} + ENVIRONMENT: ${env} \ No newline at end of file diff --git a/bhasai/nodehub/sample.env b/bhasai/nodehub/sample.env new file mode 100644 index 0000000..dec1330 --- /dev/null +++ b/bhasai/nodehub/sample.env @@ -0,0 +1,3 @@ +#nodehub +REGISTRY_BASE_URL= +env= \ No newline at end of file diff --git a/bhasai/orchestrator/docker-compose.yaml b/bhasai/orchestrator/docker-compose.yaml new file mode 100644 index 0000000..cd02821 --- /dev/null +++ b/bhasai/orchestrator/docker-compose.yaml @@ -0,0 +1,9 @@ +services: + orchestrator: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/orchestrator:${ORCHESTRATOR_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + mem_limit: ${ORCHESTRATOR_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${ORCHESTRATOR_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + restart: always + environment: + OUTBOUND_SERVICE_BASE_URL: "http://inboundjs:3000/outbound" + ENVIRONMENT: ${env} diff --git a/bhasai/orchestrator/sample.env b/bhasai/orchestrator/sample.env new file mode 100644 index 0000000..1cc8eed --- /dev/null +++ b/bhasai/orchestrator/sample.env @@ -0,0 +1,2 @@ +#orchestrator +env= \ No newline at end of file diff --git a/bhasai/pdf_parser/Caddyfile b/bhasai/pdf_parser/Caddyfile new file mode 100644 index 0000000..43b8549 --- /dev/null +++ b/bhasai/pdf_parser/Caddyfile @@ -0,0 +1,3 @@ +{$DOMAIN_NAME}://pdf-parser.{$DOMAIN_NAME} { + reverse_proxy pdf_parser:8000 +} diff --git a/bhasai/pdf_parser/docker-compose.yaml b/bhasai/pdf_parser/docker-compose.yaml new file mode 100644 index 0000000..f82e4ba --- /dev/null +++ b/bhasai/pdf_parser/docker-compose.yaml @@ -0,0 +1,82 @@ +services: + pdf_parser: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/pdf-parser:${PDF_PARSER_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + mem_limit: ${PDF_PARSER_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${PDF_PARSER_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + command: bash -c "./setup.sh && python manage.py runserver 0.0.0.0:8000" + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [gpu] + restart: always + environment: + OPENAI_API_KEY: ${OPENAI_API_KEY} + BHASHINI_API_KEY: ${BHASHINI_API_KEY} + WEBHOOK_URL: http://bff:3000/document/processed-webhook + FASTAPI_HOST: http://pdf_parser:8000 + TESSDATA_PREFIX: "/usr/share/tesseract-ocr/5/tessdata" + MINIO_BASE_URL: ${DOMAIN_SCHEME:-https}://cdn-api.${DOMAIN_NAME} + MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} + MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} + MINIO_BUCKET: pdf_parser + MINIO_SECURE_CONN: "True" + ENABLE_TRANSLATE: ${ENABLE_TRANSLATE} + ENABLE_SUMMARIZE: ${ENABLE_SUMMARIZE} + ENABLE_IMAGE_PARSING: ${ENABLE_IMAGE_PARSING} + SPLIT: ${SPLIT} + PARALLEL_FACTOR: ${PARALLEL_FACTOR} + REDIS_CONN: redis://redis:6379/0 + TORCH_DEVICE: ${TORCH_DEVICE} + CELERY_BROKER_URL: redis://redis:6379/0 + CELERY_RESULT_BACKEND: redis://redis:6379/0 + DISCORD_WEBHOOK_URL: ${DISCORD_WEBHOOK_URL} + PDF_PARSER_MAX_RETRIES : ${PDF_PARSER_MAX_RETRIES} + PDF_PARSER_SECRET_KEY : ${PDF_PARSER_SECRET_KEY} + AZURE_TRANSLATE_KEY : ${AZURE_TRANSLATE_KEY} + TRANSLATION_SERVICES : ${TRANSLATION_SERVICES} + depends_on: + - redis + + pdf_parser_celery_worker: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/pdf-parser:${PDF_PARSER_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + command: bash -c "./setup.sh && celery -A pdf_parser worker -P solo -l info" + restart: always + mem_limit: ${PDF_PARSER_CELERY_WORKER_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${PDF_PARSER_CELERY_WORKER_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [gpu] + environment: + CELERY_BROKER_URL: redis://redis:6379/0 + CELERY_RESULT_BACKEND: redis://redis:6379/0 + OPENAI_API_KEY: ${OPENAI_API_KEY} + BHASHINI_API_KEY: ${BHASHINI_API_KEY} + WEBHOOK_URL: http://bff:3000/document/processed-webhook + FASTAPI_HOST: http://pdf_parser:8000 + TESSDATA_PREFIX: "/usr/share/tesseract-ocr/5/tessdata" + MINIO_BASE_URL: ${DOMAIN_SCHEME:-https}://cdn-api.${DOMAIN_NAME} + MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} + MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} + MINIO_BUCKET: pdf_parser + MINIO_SECURE_CONN: "True" + ENABLE_TRANSLATE: ${ENABLE_TRANSLATE} + ENABLE_SUMMARIZE: ${ENABLE_SUMMARIZE} + ENABLE_IMAGE_PARSING: ${ENABLE_IMAGE_PARSING} + SPLIT: ${SPLIT} + PARALLEL_FACTOR: ${PARALLEL_FACTOR} + REDIS_CONN: redis://redis:6379/0 + TORCH_DEVICE: ${TORCH_DEVICE} + DISCORD_WEBHOOK_URL: ${DISCORD_WEBHOOK_URL} + PDF_PARSER_MAX_RETRIES : ${PDF_PARSER_MAX_RETRIES} + PDF_PARSER_SECRET_KEY : ${PDF_PARSER_SECRET_KEY} + AZURE_TRANSLATE_KEY : ${AZURE_TRANSLATE_KEY} + TRANSLATION_SERVICES : ${TRANSLATION_SERVICES} + depends_on: + - redis \ No newline at end of file diff --git a/bhasai/pdf_parser/sample.env b/bhasai/pdf_parser/sample.env new file mode 100644 index 0000000..7facf66 --- /dev/null +++ b/bhasai/pdf_parser/sample.env @@ -0,0 +1,37 @@ +#pdf_parser +OPENAI_API_KEY= +BHASHINI_API_KEY= +DOMAIN_NAME= +DOMAIN_SCHEME= +MINIO_ACCESS_KEY= +MINIO_SECRET_KEY= +ENABLE_TRANSLATE= +ENABLE_SUMMARIZE= +ENABLE_IMAGE_PARSING= +SPLIT= +PARALLEL_FACTOR= +TORCH_DEVICE= +DISCORD_WEBHOOK_URL= +PDF_PARSER_MAX_RETRIES= +PDF_PARSER_SECRET_KEY= +AZURE_TRANSLATE_KEY= +TRANSLATION_SERVICES= + +#pdf_parser_celery_worker +OPENAI_API_KEY= +BHASHINI_API_KEY= +DOMAIN_NAME= +DOMAIN_SCHEME= +MINIO_ACCESS_KEY= +MINIO_SECRET_KEY= +ENABLE_TRANSLATE= +ENABLE_SUMMARIZE= +ENABLE_IMAGE_PARSING= +SPLIT= +PARALLEL_FACTOR= +TORCH_DEVICE= +DISCORD_WEBHOOK_URL= +PDF_PARSER_MAX_RETRIES= +PDF_PARSER_SECRET_KEY= +AZURE_TRANSLATE_KEY= +TRANSLATION_SERVICES= diff --git a/bhasai/telemetry/Caddyfile b/bhasai/telemetry/Caddyfile new file mode 100644 index 0000000..951099a --- /dev/null +++ b/bhasai/telemetry/Caddyfile @@ -0,0 +1,3 @@ +{$DOMAIN_SCHEME}://telemetry.{$DOMAIN_NAME} { + reverse_proxy telemetry:3000 +} \ No newline at end of file diff --git a/bhasai/telemetry/docker-compose.yaml b/bhasai/telemetry/docker-compose.yaml new file mode 100644 index 0000000..0a713a0 --- /dev/null +++ b/bhasai/telemetry/docker-compose.yaml @@ -0,0 +1,13 @@ +services: + telemetry: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/telemetry:${TELEMETRY_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + restart: always + mem_limit: ${TELEMETRY_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${TELEMETRY_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + environment: + DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/telemetry" + CLICKHOUSE_HOST: http://clickhouse:8123 + CLICKHOUSE_DB: ${CLICKHOUSE_DB} + CLICKHOUSE_USER: ${CLICKHOUSE_USER} + CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD} + AUTH_SERVICE_URL: http://auth_service:3000 diff --git a/bhasai/telemetry/sample.env b/bhasai/telemetry/sample.env new file mode 100644 index 0000000..774e9cb --- /dev/null +++ b/bhasai/telemetry/sample.env @@ -0,0 +1,6 @@ +#telemetry +POSTGRES_USER= +POSTGRES_PASSWORD= +CLICKHOUSE_DB= +CLICKHOUSE_USER= +CLICKHOUSE_PASSWORD= \ No newline at end of file diff --git a/bhasai/transport_socket/Caddyfile b/bhasai/transport_socket/Caddyfile new file mode 100644 index 0000000..311c310 --- /dev/null +++ b/bhasai/transport_socket/Caddyfile @@ -0,0 +1,3 @@ +{$DOMAIN_SCHEME}://transport-socket.{$DOMAIN_NAME} { + reverse_proxy transport_socket:3000 +} diff --git a/bhasai/transport_socket/docker-compose.yaml b/bhasai/transport_socket/docker-compose.yaml new file mode 100644 index 0000000..31b51eb --- /dev/null +++ b/bhasai/transport_socket/docker-compose.yaml @@ -0,0 +1,12 @@ +services: + transport_socket: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/transport-socket:${TRANSPORT_SOCKET_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + restart: always + mem_limit: ${TRANSPORT_SOCKET_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${TRANSPORT_SOCKET_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + environment: + JWKS_URI: http://fusionauth:9011/.well-known/jwks.json + REDIS_URL: redis://redis:6379/1 + ADAPTER_URL: http://inboundjs:3000/inbound/bot + DEBUG: ${TRANSPORT_SOCKET_DEBUG} + SERVER_PORT: 3000 \ No newline at end of file diff --git a/bhasai/transport_socket/sample.env b/bhasai/transport_socket/sample.env new file mode 100644 index 0000000..bd1ae4b --- /dev/null +++ b/bhasai/transport_socket/sample.env @@ -0,0 +1,2 @@ +#transport_socket +TRANSPORT_SOCKET_DEBUG= \ No newline at end of file diff --git a/bhasai/uci_apis/Caddyfile b/bhasai/uci_apis/Caddyfile new file mode 100644 index 0000000..69ef4c0 --- /dev/null +++ b/bhasai/uci_apis/Caddyfile @@ -0,0 +1,3 @@ +{$DOMAIN_SCHEME}://uci-apis.{$DOMAIN_NAME} { + reverse_proxy uci_apis:3000 +} \ No newline at end of file diff --git a/bhasai/uci_apis/docker-compose.yaml b/bhasai/uci_apis/docker-compose.yaml new file mode 100644 index 0000000..2f1443b --- /dev/null +++ b/bhasai/uci_apis/docker-compose.yaml @@ -0,0 +1,19 @@ +services: + uci_apis: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/uci-apis:${UCI_APIS_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + restart: always + mem_limit: ${UCI_APIS_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${UCI_APIS_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + depends_on: + db: + condition: service_healthy + environment: + UCI_API_PSQL_DB_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/uci_apis" + REDIS_URL: redis://redis:6379/2 + FUSIONAUTH_URL: http://fusionauth:9011 + FUSIONAUTH_KEY: ${FUSIONAUTH_API_KEY} + ADMIN_TOKEN: ${UCI_ADMIN_TOKEN} + VAULT_ADDR: http://vault:8200 + VAULT_USERNAME: ${VAULT_USERNAME} + VAULT_PASSWORD: ${VAULT_PASSWORD} + AUTH_SERVICE_URL: http://auth_service:3000 \ No newline at end of file diff --git a/bhasai/uci_apis/sample.env b/bhasai/uci_apis/sample.env new file mode 100644 index 0000000..9810c0d --- /dev/null +++ b/bhasai/uci_apis/sample.env @@ -0,0 +1,8 @@ + +#uci_apis +UCI_ADMIN_TOKEN= +VAULT_USERNAME= +VAULT_PASSWORD= +POSTGRES_USER= +POSTGRES_PASSWORD= +FUSIONAUTH_API_KEY= \ No newline at end of file diff --git a/bhasai/user_management_service/Caddyfile b/bhasai/user_management_service/Caddyfile new file mode 100644 index 0000000..05a232c --- /dev/null +++ b/bhasai/user_management_service/Caddyfile @@ -0,0 +1,3 @@ +{$DOMAIN_NAME}://user-management-service.{$DOMAIN_NAME} { + reverse_proxy user_management_service:3000 +} \ No newline at end of file diff --git a/bhasai/user_management_service/docker-compose.yaml b/bhasai/user_management_service/docker-compose.yaml new file mode 100644 index 0000000..b3e0afa --- /dev/null +++ b/bhasai/user_management_service/docker-compose.yaml @@ -0,0 +1,11 @@ +services: + user_management_service: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/user-management-service:${USER_MANAGEMENT_SERVICE_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + restart: always + mem_limit: ${USER_MANAGEMENT_SERVICE_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${USER_MANAGEMENT_SERVICE_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + depends_on: + db: + condition: service_healthy + environment: + DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/user_management_service" diff --git a/bhasai/user_management_service/sample.env b/bhasai/user_management_service/sample.env new file mode 100644 index 0000000..bc99394 --- /dev/null +++ b/bhasai/user_management_service/sample.env @@ -0,0 +1,4 @@ + +#user_management_service +POSTGRES_USER= +POSTGRES_PASSWORD= \ No newline at end of file diff --git a/bhasai/vault/Dockerfile b/bhasai/vault/Dockerfile new file mode 100644 index 0000000..41acb0e --- /dev/null +++ b/bhasai/vault/Dockerfile @@ -0,0 +1,4 @@ +FROM hashicorp/consul-template:0.37.2 as consul-template +FROM vault:1.13.3 +RUN apk add bash jq +COPY --from=consul-template /bin/consul-template /bin/consul-template \ No newline at end of file diff --git a/bhasai/vault/admin-policy.hcl b/bhasai/vault/admin-policy.hcl new file mode 100644 index 0000000..3609b94 --- /dev/null +++ b/bhasai/vault/admin-policy.hcl @@ -0,0 +1,59 @@ +# Read system health check +path "sys/health" +{ + capabilities = ["read", "sudo"] +} + +# Create and manage ACL policies broadly across Vault + +# List existing policies +path "sys/policies/acl" +{ + capabilities = ["list"] +} + +# Create and manage ACL policies +path "sys/policies/acl/*" +{ + capabilities = ["create", "read", "update", "delete", "list", "sudo"] +} + +# Enable and manage authentication methods broadly across Vault + +# Manage auth methods broadly across Vault +path "auth/*" +{ + capabilities = ["create", "read", "update", "delete", "list", "sudo"] +} + +# Create, update, and delete auth methods +path "sys/auth/*" +{ + capabilities = ["create", "update", "delete", "sudo"] +} + +# List auth methods +path "sys/auth" +{ + capabilities = ["read"] +} + +# Enable and manage the key/value secrets engine at `secret/` path + +# List, create, update, and delete key/value secrets +path "kv/*" +{ + capabilities = ["create", "read", "update", "delete", "list", "sudo"] +} + +# Manage secrets engines +path "sys/mounts/*" +{ + capabilities = ["create", "read", "update", "delete", "list", "sudo"] +} + +# List existing secrets engines. +path "sys/mounts" +{ + capabilities = ["read"] +} \ No newline at end of file diff --git a/bhasai/vault/consul-init.sh b/bhasai/vault/consul-init.sh new file mode 100755 index 0000000..dc79205 --- /dev/null +++ b/bhasai/vault/consul-init.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +set -e +pushd /bhasai + +export VAULT_ADDR="http://127.0.0.1:8200" + +# Authenticate using username and password +vault login -method=userpass username=$VAULT_USERNAME password=$VAULT_PASSWORD + +export VAULT_TOKEN=$(vault print token) + +export IFS=$'\n' +if ! vault kv get kv/env >/dev/null 2>&1; then + vault kv put kv/env \ + $(grep -v '^#' .env | grep -e '[^[:space:]]' | \ + sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' | \ + awk -F '=' '{gsub(/^["'\'']|["'\'']$/, "", $2); data[$1]=$2} END {for (key in data) print key "=" data[key]}') +# create a backup of the original .env +cp .env .env.bak +else + # Backup the last file created by Consul + if [ -f env.tmp ]; then + mv env.tmp .env.back + fi +fi + + +consul-template \ + -vault-default-lease-duration=10s \ + -vault-renew-token=false \ + -template "env.tpl:env.tmp:bash -c 'cat /bhasai/env.tmp > /bhasai/.env; echo .env updated '" diff --git a/bhasai/vault/docker-compose.yaml b/bhasai/vault/docker-compose.yaml new file mode 100644 index 0000000..99592c2 --- /dev/null +++ b/bhasai/vault/docker-compose.yaml @@ -0,0 +1,22 @@ +volumes: + environment: + +services: + vault: + build: + context: . + dockerfile: Dockerfile + restart: always + environment: + VAULT_USERNAME: ${VAULT_USERNAME:?VAULT_USERNAME is not set} + VAULT_PASSWORD: ${VAULT_PASSWORD:?VAULT_PASSWORD is not set} + volumes: + - environment:/vault/file + - ./vault.json:/vault/config/vault.json + - ./admin-policy.hcl:/vault/policies/admin-policy.hcl + - ./vault-init.sh:/vault-init.sh + - ../../.env:/bhasai/.env + - ./:/bhasai/ + cap_add: + - IPC_LOCK + command: sh -c "bash vault-init.sh" diff --git a/bhasai/vault/env.tpl b/bhasai/vault/env.tpl new file mode 100644 index 0000000..fe7e39b --- /dev/null +++ b/bhasai/vault/env.tpl @@ -0,0 +1,2 @@ +{{ with secret "kv/env" }} {{ range $k, $v := .Data.data }} +{{ printf "%s='%s'" $k $v | trimSpace }}{{ end }}{{ end }} \ No newline at end of file diff --git a/bhasai/vault/vault-init.sh b/bhasai/vault/vault-init.sh new file mode 100644 index 0000000..1f96f79 --- /dev/null +++ b/bhasai/vault/vault-init.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +set -me + +if [[ -z $VAULT_USERNAME && -z $VAULT_PASSWORD ]] ; then + echo "VAULT_USERNAME and VAULT_PASSWORD are not set" + exit 1 +fi + +vault server -config=/vault/config/vault.json & + +sleep 2 + +export VAULT_ADDR="http://127.0.0.1:8200" +VAULT_HEALTH_URL="$VAULT_ADDR/v1/sys/seal-status" +while true; do + response=$(wget -qO- "$VAULT_HEALTH_URL") + + # Check the exit status of wget to handle errors + if [ $? -ne 0 ]; then + echo "Waiting for Vault to be ready" + else + initialized=$(echo "$response" | jq -r '.initialized') + sealed=$(echo "$response" | jq -r '.sealed') + echo "Vault Status - Initialized: $initialized, Sealed: $sealed" + break + fi + + sleep 2 +done + +if [ "$initialized" = false ]; then +vault operator init > /vault/file/generated_keys.txt +fi +if [ "$sealed" = true ]; then +# Parse unsealed keys +while IFS= read -r line; do + key=$(echo "$line" | cut -c15-) + if [ -n "$key" ]; then + vault operator unseal "$key" + fi +done < <(grep "Unseal Key " /vault/file/generated_keys.txt) +fi + + +# Get root token +VAULT_TOKEN=$(grep "Initial Root Token: " < /vault/file/generated_keys.txt | cut -c21- ) +export VAULT_TOKEN + +if vault secrets list | grep -q "kv/"; then + echo "KV version 2 secrets engine is already enabled." +else + vault secrets enable -version=2 kv +fi + +if ! vault auth list | grep -q "userpass/"; then + vault auth enable userpass +else + echo "userpass authentication method is already enabled." +fi + +if ! vault policy list | grep -q "admin-policy"; then + + vault policy write admin-policy /vault/policies/admin-policy.hcl +else + echo "admin-policy already exists." +fi + + +if ! vault read auth/userpass/users/"$VAULT_USERNAME" > /dev/null 2>&1; then + vault write auth/userpass/users/"$VAULT_USERNAME" \ + password="$VAULT_PASSWORD" \ + policies=admin-policy +else + echo "User $VAULT_USERNAME already exists." +fi + +/bhasai/consul-init.sh & + +fg %1 diff --git a/bhasai/vault/vault.json b/bhasai/vault/vault.json new file mode 100644 index 0000000..0280b2a --- /dev/null +++ b/bhasai/vault/vault.json @@ -0,0 +1,14 @@ +{ +"backend": { + "file": { + "path": "/vault/file" + } + }, + "listener": { + "tcp": { + "address": "0.0.0.0:8200", + "tls_disable": 1 + } + }, + "ui": true +} \ No newline at end of file diff --git a/bhasai/yt_parser/Caddyfile b/bhasai/yt_parser/Caddyfile new file mode 100644 index 0000000..9667bfc --- /dev/null +++ b/bhasai/yt_parser/Caddyfile @@ -0,0 +1,3 @@ +{$DOMAIN_NAME}://yt-parser.{$DOMAIN_NAME} { + reverse_proxy yt_parser:8000 +} \ No newline at end of file diff --git a/bhasai/yt_parser/docker-compose.yaml b/bhasai/yt_parser/docker-compose.yaml new file mode 100644 index 0000000..6744591 --- /dev/null +++ b/bhasai/yt_parser/docker-compose.yaml @@ -0,0 +1,51 @@ +services: + yt_parser: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/yt-parser:${YT_PARSER_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + command: uvicorn main:app --host 0.0.0.0 --port 8000 + restart: always + mem_limit: ${YT_PARSER_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${YT_PARSER_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + environment: + MINIO_BASE_URL: "minio:9000" + MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} + MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} + MINIO_BUCKET: yt_parser + MINIO_SECURE_CONN: "False" + REDIS_CONN: redis://redis:6379/3 + CELERY_BROKER_URL: redis://redis:6379/3 + CELERY_RESULT_BACKEND: redis://redis:6379/3 + BHASHINI_USERID: ${BHASHINI_USERID} + BHASHINI_ULCA_API_KEY: ${BHASHINI_ULCA_API_KEY} + AZURE_SUBSCRIPTION_KEY: ${AZURE_SUBSCRIPTION_KEY} + WEBHOOK_URL: http://bff:3000/document/processed-webhook/youtube + DISCORD_WEBHOOK_URL: ${DISCORD_WEBHOOK_URL} + ASR_SERVICE_PRIORITY: ${ASR_SERVICE_PRIORITY} + YOUTUBE_DATA_API_KEY: ${YOUTUBE_DATA_API_KEY} + depends_on: + - redis + + yt_parser_celery_worker: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/yt-parser:${YT_PARSER_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + command: celery -A worker worker -P gevent --loglevel=info + restart: always + mem_limit: ${YT_PARSER_CELERY_WORKER_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${YT_PARSER_CELERY_WORKER_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + environment: + CELERY_BROKER_URL: redis://redis:6379/3 + CELERY_RESULT_BACKEND: redis://redis:6379/3 + WEBHOOK_URL: http://bff:3000/document/processed-webhook/youtube + MINIO_BASE_URL: "minio:9000" + MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} + MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} + MINIO_BUCKET: yt_parser + MINIO_SECURE_CONN: "False" + REDIS_CONN: redis://redis:6379/3 + BHASHINI_USERID: ${BHASHINI_USERID} + BHASHINI_ULCA_API_KEY: ${BHASHINI_ULCA_API_KEY} + AZURE_SUBSCRIPTION_KEY: ${AZURE_SUBSCRIPTION_KEY} + DISCORD_WEBHOOK_URL: ${DISCORD_WEBHOOK_URL} + ASR_SERVICE_PRIORITY: ${ASR_SERVICE_PRIORITY} + YOUTUBE_DATA_API_KEY: ${YOUTUBE_DATA_API_KEY} + depends_on: + - redis + \ No newline at end of file diff --git a/bhasai/yt_parser/sample.env b/bhasai/yt_parser/sample.env new file mode 100644 index 0000000..3357bcc --- /dev/null +++ b/bhasai/yt_parser/sample.env @@ -0,0 +1,19 @@ +#yt_parser +MINIO_ACCESS_KEY= +MINIO_SECRET_KEY= +BHASHINI_USERID= +BHASHINI_ULCA_API_KEY= +AZURE_SUBSCRIPTION_KEY= +DISCORD_WEBHOOK_URL= +ASR_SERVICE_PRIORITY= +YOUTUBE_DATA_API_KEY= + +#yt_parser_celery_worker +MINIO_ACCESS_KEY= +MINIO_SECRET_KEY= +BHASHINI_USERID= +BHASHINI_ULCA_API_KEY= +AZURE_SUBSCRIPTION_KEY= +DISCORD_WEBHOOK_URL= +ASR_SERVICE_PRIORITY= +YOUTUBE_DATA_API_KEY= From a41ea33315efa9506921e68b5fc6bcd858ed64e6 Mon Sep 17 00:00:00 2001 From: Karun Agarwal Date: Thu, 1 Aug 2024 18:23:27 +0530 Subject: [PATCH 02/28] added Caddyfile and docker-compose.yaml in root and in bhasai --- Caddyfile | 8 ++++++++ bhasai/Caddyfile | 16 ++++++++++++++++ bhasai/docker-compose.yaml | 19 +++++++++++++++++++ docker-compose.yaml | 25 +++++++++++++++++++++++++ 4 files changed, 68 insertions(+) create mode 100644 Caddyfile create mode 100644 bhasai/Caddyfile create mode 100644 bhasai/docker-compose.yaml create mode 100644 docker-compose.yaml diff --git a/Caddyfile b/Caddyfile new file mode 100644 index 0000000..6355cff --- /dev/null +++ b/Caddyfile @@ -0,0 +1,8 @@ +import ./common/monitoring/Caddyfile +import ./common/minio/Caddyfile +import ./common/environment/Caddyfile +import ./common/fusionauth/Caddyfile +# The registry doesn't have a auth thus exposing it publicly means anyone can access the images pushed to this registry +# import ./common/registry/Caddyfile +import ./common/uptime/Caddyfile +import ./bhasai/Caddyfile \ No newline at end of file diff --git a/bhasai/Caddyfile b/bhasai/Caddyfile new file mode 100644 index 0000000..b5ff724 --- /dev/null +++ b/bhasai/Caddyfile @@ -0,0 +1,16 @@ +import ./admin/Caddyfile +import ./ai_tools_proxy/Caddyfile +import ./auth_service/Caddyfile +import ./autotune/Caddyfile +import ./bff/Caddyfile +import ./dataset_service/Caddyfile +import ./deployer/Caddyfile +import ./document_service/Caddyfile +import ./inboundjs/Caddyfile +import ./nodehub/Caddyfile +import ./pdf_parser/Caddyfile +import ./telemetry/Caddyfile +import ./transport_socket/Caddyfile +import ./uci_apis/Caddyfile +import ./user_management_service/Caddyfile +import ./yt_parser/Caddyfile \ No newline at end of file diff --git a/bhasai/docker-compose.yaml b/bhasai/docker-compose.yaml new file mode 100644 index 0000000..ab2cdff --- /dev/null +++ b/bhasai/docker-compose.yaml @@ -0,0 +1,19 @@ +include: + - ./admin/docker-compose.yaml + - ./ai_tools_proxy/docker-compose.yaml + - ./auth_service/docker-compose.yaml + - ./autotune/docker-compose.yaml + - ./bff/docker-compose.yaml + - ./dataset_service/docker-compose.yaml + - ./deployer/docker-compose.yaml + - ./document_service/docker-compose.yaml + - ./inboundjs/docker-compose.yaml + - ./nodehub/docker-compose.yaml + - ./orchestrator/docker-compose.yaml + - ./pdf_parser/docker-compose.yaml + - ./telemetry/docker-compose.yaml + - ./transport_socket/docker-compose.yaml + - ./uci_apis/docker-compose.yaml + - ./user_management_service/docker-compose.yaml + - ./vault/docker-compose.yaml + - ./yt_parser/docker-compose.yaml diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..c86d82c --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,25 @@ +include: + - ./common/monitoring/docker-compose.yaml + - ./common/minio/docker-compose.yaml + - ./common/environment/docker-compose.yaml + - ./common/fusionauth/docker-compose.yaml + - ./common/uptime/docker-compose.yaml + - ./common/clickhouse/docker-compose.yaml + - ./common/redis/docker-compose.yaml + - ./common/db/docker-compose.yaml +# - ./common/registry/docker-compose.yaml + - ./bhasai/docker-compose.yaml + + +services: + caddy: + extends: + file: common/caddy/docker-compose.yaml + service: caddy +# volumes: +# - project-specific-volume-mappings:/etc/caddy/project-name + + +volumes: + caddy_data: + caddy_config: From 8149f347412c53a1fd373ad030b342af8478c732 Mon Sep 17 00:00:00 2001 From: Srijan Srivastava Date: Fri, 2 Aug 2024 15:26:10 +0530 Subject: [PATCH 03/28] added SDC flavour for docker-compose.yaml and Caddyfile (#26) * added SDC flavour for docker-compose.yaml and Caddyfile --------- Co-authored-by: Karun Agarwal <113603846+singhalkarun@users.noreply.github.com> --- Caddyfile.SDC | 7 ++ bhasai/Caddyfile.SDC | 16 ++++ bhasai/admin/Caddyfile.SDC | 3 + bhasai/admin/docker-compose.SDC.yaml | 16 ++++ bhasai/admin/docker-compose.yaml | 2 +- bhasai/ai_tools_proxy/Caddyfile.SDC | 3 + bhasai/ai_tools_proxy/docker-compose.SDC.yaml | 22 +++++ bhasai/auth_service/Caddyfile.SDC | 3 + bhasai/auth_service/docker-compose.SDC.yaml | 9 ++ bhasai/autotune/Caddyfile.SDC | 3 + bhasai/autotune/docker-compose.SDC.yaml | 49 +++++++++++ bhasai/bff/Caddyfile.SDC | 3 + bhasai/bff/docker-compose.SDC.yaml | 48 +++++++++++ bhasai/dataset_service/Caddyfile.SDC | 3 + .../dataset_service/docker-compose.SDC.yaml | 19 +++++ bhasai/deployer/Caddyfile.SDC | 3 + bhasai/deployer/docker-compose.SDC.yaml | 20 +++++ bhasai/docker-compose.SDC.yaml | 19 +++++ bhasai/document_service/Caddyfile.SDC | 3 + .../document_service/docker-compose.SDC.yaml | 22 +++++ bhasai/inboundjs/Caddyfile.SDC | 3 + bhasai/inboundjs/docker-compose.SDC.yaml | 13 +++ bhasai/nodehub/Caddyfile.SDC | 3 + bhasai/nodehub/docker-compose.SDC.yaml | 11 +++ bhasai/orchestrator/docker-compose.SDC.yaml | 9 ++ bhasai/pdf_parser/Caddyfile.SDC | 3 + bhasai/pdf_parser/docker-compose.SDC.yaml | 82 +++++++++++++++++++ bhasai/telemetry/Caddyfile.SDC | 3 + bhasai/telemetry/docker-compose.SDC.yaml | 13 +++ bhasai/transport_socket/Caddyfile.SDC | 3 + .../transport_socket/docker-compose.SDC.yaml | 12 +++ bhasai/uci_apis/Caddyfile.SDC | 3 + bhasai/uci_apis/docker-compose.SDC.yaml | 19 +++++ bhasai/user_management_service/Caddyfile.SDC | 3 + .../docker-compose.SDC.yaml | 11 +++ bhasai/yt_parser/Caddyfile.SDC | 3 + bhasai/yt_parser/docker-compose.SDC.yaml | 51 ++++++++++++ docker-compose.SDC.yaml | 24 ++++++ 38 files changed, 541 insertions(+), 1 deletion(-) create mode 100644 Caddyfile.SDC create mode 100644 bhasai/Caddyfile.SDC create mode 100644 bhasai/admin/Caddyfile.SDC create mode 100644 bhasai/admin/docker-compose.SDC.yaml create mode 100644 bhasai/ai_tools_proxy/Caddyfile.SDC create mode 100644 bhasai/ai_tools_proxy/docker-compose.SDC.yaml create mode 100644 bhasai/auth_service/Caddyfile.SDC create mode 100644 bhasai/auth_service/docker-compose.SDC.yaml create mode 100644 bhasai/autotune/Caddyfile.SDC create mode 100644 bhasai/autotune/docker-compose.SDC.yaml create mode 100644 bhasai/bff/Caddyfile.SDC create mode 100644 bhasai/bff/docker-compose.SDC.yaml create mode 100644 bhasai/dataset_service/Caddyfile.SDC create mode 100644 bhasai/dataset_service/docker-compose.SDC.yaml create mode 100644 bhasai/deployer/Caddyfile.SDC create mode 100644 bhasai/deployer/docker-compose.SDC.yaml create mode 100644 bhasai/docker-compose.SDC.yaml create mode 100644 bhasai/document_service/Caddyfile.SDC create mode 100644 bhasai/document_service/docker-compose.SDC.yaml create mode 100644 bhasai/inboundjs/Caddyfile.SDC create mode 100644 bhasai/inboundjs/docker-compose.SDC.yaml create mode 100644 bhasai/nodehub/Caddyfile.SDC create mode 100644 bhasai/nodehub/docker-compose.SDC.yaml create mode 100644 bhasai/orchestrator/docker-compose.SDC.yaml create mode 100644 bhasai/pdf_parser/Caddyfile.SDC create mode 100644 bhasai/pdf_parser/docker-compose.SDC.yaml create mode 100644 bhasai/telemetry/Caddyfile.SDC create mode 100644 bhasai/telemetry/docker-compose.SDC.yaml create mode 100644 bhasai/transport_socket/Caddyfile.SDC create mode 100644 bhasai/transport_socket/docker-compose.SDC.yaml create mode 100644 bhasai/uci_apis/Caddyfile.SDC create mode 100644 bhasai/uci_apis/docker-compose.SDC.yaml create mode 100644 bhasai/user_management_service/Caddyfile.SDC create mode 100644 bhasai/user_management_service/docker-compose.SDC.yaml create mode 100644 bhasai/yt_parser/Caddyfile.SDC create mode 100644 bhasai/yt_parser/docker-compose.SDC.yaml create mode 100644 docker-compose.SDC.yaml diff --git a/Caddyfile.SDC b/Caddyfile.SDC new file mode 100644 index 0000000..bca8b98 --- /dev/null +++ b/Caddyfile.SDC @@ -0,0 +1,7 @@ +{$DOMAIN_SCHEME}://{$DOMAIN_NAME} { + {$TLS_ENTRY} + + import ./common/monitoring/Caddyfile.SDC + import ./bhasai/Caddyfile.SDC + +} \ No newline at end of file diff --git a/bhasai/Caddyfile.SDC b/bhasai/Caddyfile.SDC new file mode 100644 index 0000000..ef1e1f9 --- /dev/null +++ b/bhasai/Caddyfile.SDC @@ -0,0 +1,16 @@ +import ./admin/Caddyfile.SDC +import ./ai_tools_proxy/Caddyfile.SDC +import ./auth_service/Caddyfile.SDC +import ./autotune/Caddyfile.SDC +import ./bff/Caddyfile.SDC +import ./dataset_service/Caddyfile.SDC +import ./deployer/Caddyfile.SDC +import ./document_service/Caddyfile.SDC +import ./inboundjs/Caddyfile.SDC +import ./nodehub/Caddyfile.SDC +import ./pdf_parser/Caddyfile.SDC +import ./telemetry/Caddyfile.SDC +import ./transport_socket/Caddyfile.SDC +import ./uci_apis/Caddyfile.SDC +import ./user_management_service/Caddyfile.SDC +import ./yt_parser/Caddyfile.SDC \ No newline at end of file diff --git a/bhasai/admin/Caddyfile.SDC b/bhasai/admin/Caddyfile.SDC new file mode 100644 index 0000000..661275a --- /dev/null +++ b/bhasai/admin/Caddyfile.SDC @@ -0,0 +1,3 @@ +handle_path /admin* { + reverse_proxy admin:3000 +} diff --git a/bhasai/admin/docker-compose.SDC.yaml b/bhasai/admin/docker-compose.SDC.yaml new file mode 100644 index 0000000..0ea724a --- /dev/null +++ b/bhasai/admin/docker-compose.SDC.yaml @@ -0,0 +1,16 @@ +services: + admin: + build: + context: https://${GIT_USERNAME}:${GIT_PASSWORD}@github.com/${org}/admin.git#${ADMIN_GITHUB_BRANCH:-${DEFAULT_GITHUB_BRANCH:?DEFAULT_GITHUB_BRANCH is not set}} + args: + NEXT_PUBLIC_UCI_API_URL: ${DOMAIN_SCHEME:-https}://${DOMAIN_NAME}/uci-apis + NEXT_PUBLIC_BFF_API_URL: ${DOMAIN_SCHEME:-https}://${DOMAIN_NAME}/bff + NEXT_PUBLIC_SOCKET_URL: ${DOMAIN_SCHEME:-https}://{DOMAIN_NAME}/transport-socket + NEXT_PUBLIC_AUTH_SERVICE_URL: ${DOMAIN_SCHEME:-https}://${DOMAIN_NAME}/auth-service + NEXT_PUBLIC_DATASET_SERVICE_URL: ${DOMAIN_SCHEME:-https}://${DOMAIN_NAME}/dataset-service + NEXT_PUBLIC_DOC_SERVICE_URL: ${DOMAIN_SCHEME:-https}://${DOMAIN_NAME}/document-service + NEXT_PUBLIC_PWA_DEPLOYER_URL: ${DOMAIN_SCHEME:-https}://${DOMAIN_NAME}/deployer + NEXT_PUBLIC_TELEMETRY_API_URL: ${DOMAIN_SCHEME:-https}://${DOMAIN_NAME}/telemetry + NEXT_PUBLIC_FLOWISE_URL: ${NEXT_PUBLIC_FLOWISE_URL} + NEXT_PUBLIC_AUTOTUNE_API_URL: ${DOMAIN_SCHEME:-https}://${DOMAIN_NAME}/autotune + restart: always diff --git a/bhasai/admin/docker-compose.yaml b/bhasai/admin/docker-compose.yaml index 0af09fd..1151d52 100644 --- a/bhasai/admin/docker-compose.yaml +++ b/bhasai/admin/docker-compose.yaml @@ -5,7 +5,7 @@ services: args: NEXT_PUBLIC_UCI_API_URL: ${DOMAIN_SCHEME:-https}://uci-apis.${DOMAIN_NAME} NEXT_PUBLIC_BFF_API_URL: ${DOMAIN_SCHEME:-https}://bff.${DOMAIN_NAME} - NEXT_PUBLIC_SOCKET_URL: ${DOMAIN_SCHEME:-https}://ts.${DOMAIN_NAME} + NEXT_PUBLIC_SOCKET_URL: ${DOMAIN_SCHEME:-https}://transport-socket.${DOMAIN_NAME} NEXT_PUBLIC_AUTH_SERVICE_URL: ${DOMAIN_SCHEME:-https}://auth-service.${DOMAIN_NAME} NEXT_PUBLIC_DATASET_SERVICE_URL: ${DOMAIN_SCHEME:-https}://dataset-service.${DOMAIN_NAME} NEXT_PUBLIC_DOC_SERVICE_URL: ${DOMAIN_SCHEME:-https}://document-service.${DOMAIN_NAME} diff --git a/bhasai/ai_tools_proxy/Caddyfile.SDC b/bhasai/ai_tools_proxy/Caddyfile.SDC new file mode 100644 index 0000000..98f18bc --- /dev/null +++ b/bhasai/ai_tools_proxy/Caddyfile.SDC @@ -0,0 +1,3 @@ +handle_path /ai-tools-proxy* { + reverse_proxy ai_tools_proxy:3000 +} diff --git a/bhasai/ai_tools_proxy/docker-compose.SDC.yaml b/bhasai/ai_tools_proxy/docker-compose.SDC.yaml new file mode 100644 index 0000000..a6cefc7 --- /dev/null +++ b/bhasai/ai_tools_proxy/docker-compose.SDC.yaml @@ -0,0 +1,22 @@ + services: + ai_tools_proxy: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/ai-tools-proxy:${AI_TOOLS_PROXY_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + restart: always + mem_limit: ${AI_TOOLS_PROXY_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${AI_TOOLS_PROXY_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + environment: + DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/ai_tools_proxy" + DEFAULT_RETRY_COUNT: ${DEFAULT_RETRY_COUNT} + DEFAULT_TIMEOUT: ${DEFAULT_TIMEOUT} + ULCA_API_KEY: ${ULCA_API_KEY} + ULCA_CONFIG_URL: ${ULCA_CONFIG_URL} + ULCA_USER_ID: ${ULCA_USER_ID} + MINIO_ENDPOINT: ${DOMAIN_SCHEME:-https}://cdn-api.${DOMAIN_NAME} + MINIO_PORT: 443 + MINIO_SECURE_CONN: "True" + MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} + MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} + MINIO_BUCKET: ai-tools-proxy + ENVIRONMENT: ${env} + TELEMETRY_URL: http://telemetry:3000 + OPENAI_API_KEY: ${OPENAI_API_KEY} diff --git a/bhasai/auth_service/Caddyfile.SDC b/bhasai/auth_service/Caddyfile.SDC new file mode 100644 index 0000000..7315e70 --- /dev/null +++ b/bhasai/auth_service/Caddyfile.SDC @@ -0,0 +1,3 @@ +handle_path /auth-service* { + reverse_proxy auth_service:3000 +} diff --git a/bhasai/auth_service/docker-compose.SDC.yaml b/bhasai/auth_service/docker-compose.SDC.yaml new file mode 100644 index 0000000..f52215b --- /dev/null +++ b/bhasai/auth_service/docker-compose.SDC.yaml @@ -0,0 +1,9 @@ +services: + auth_service: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/auth-service:${AUTH_SERVICE_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + restart: always + mem_limit: ${AUTH_SERVICE_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${AUTH_SERVICE_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + environment: + FUSIONAUTH_URL: http://fusionauth:9011 + FUSIONAUTH_KEY: ${FUSIONAUTH_API_KEY} \ No newline at end of file diff --git a/bhasai/autotune/Caddyfile.SDC b/bhasai/autotune/Caddyfile.SDC new file mode 100644 index 0000000..7c636cf --- /dev/null +++ b/bhasai/autotune/Caddyfile.SDC @@ -0,0 +1,3 @@ +handle_path /autotune* { + reverse_proxy autotune:8000 +} diff --git a/bhasai/autotune/docker-compose.SDC.yaml b/bhasai/autotune/docker-compose.SDC.yaml new file mode 100644 index 0000000..00f470f --- /dev/null +++ b/bhasai/autotune/docker-compose.SDC.yaml @@ -0,0 +1,49 @@ + services: + autotune: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/autotune:${AUTOTUNE_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + restart: always + mem_limit: ${AUTOTUNE_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${AUTOTUNE_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + command: bash -c "poetry install && python manage.py database && python manage.py runserver 0.0.0.0:8000" + environment: + DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/autotune" + REDIS_URL: redis://redis:6379/7 + OPENAI_API_KEY: ${OPENAI_API_KEY} + MINIO_BASE_URL: "${DOMAIN_SCHEME:-https}://cdn-api.${DOMAIN_NAME}" + MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} + MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} + MINIO_BUCKET: autotune + MINIO_SECURE_CONN: "True" + CELERY_MAX_RETRIES: ${CELERY_MAX_RETRIES} + DJANGO_SECRET_KEY: ${DJANGO_SECRET_KEY} + HUGGING_FACE_USERNAME: ${HUGGING_FACE_USERNAME} + HUGGING_FACE_TOKEN: ${HUGGING_FACE_TOKEN} + GITHUB_PAT: ${GITHUB_PAT} + + autotune_celery_worker: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/autotune:${AUTOTUNE_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + restart: always + mem_limit: ${AUTOTUNE_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${AUTOTUNE_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + command: bash -c "poetry install && celery -A autotune worker --loglevel=info -P gevent" + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [gpu] + environment: + DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/autotune" + REDIS_URL: redis://redis:6379/7 + OPENAI_API_KEY: ${OPENAI_API_KEY} + MINIO_BASE_URL: "${DOMAIN_SCHEME:-https}://cdn-api.${DOMAIN_NAME}" + MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} + MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} + MINIO_BUCKET: autotune + MINIO_SECURE_CONN: "True" + CELERY_MAX_RETRIES: ${CELERY_MAX_RETRIES} + DJANGO_SECRET_KEY: ${DJANGO_SECRET_KEY} + HUGGING_FACE_USERNAME: ${HUGGING_FACE_USERNAME} + HUGGING_FACE_TOKEN: ${HUGGING_FACE_TOKEN} + GITHUB_PAT: ${GITHUB_PAT} diff --git a/bhasai/bff/Caddyfile.SDC b/bhasai/bff/Caddyfile.SDC new file mode 100644 index 0000000..02525b9 --- /dev/null +++ b/bhasai/bff/Caddyfile.SDC @@ -0,0 +1,3 @@ +handle_path /bff* { + reverse_proxy bff:3000 +} diff --git a/bhasai/bff/docker-compose.SDC.yaml b/bhasai/bff/docker-compose.SDC.yaml new file mode 100644 index 0000000..f96c686 --- /dev/null +++ b/bhasai/bff/docker-compose.SDC.yaml @@ -0,0 +1,48 @@ +services: + bff: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/bff:${BFF_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + restart: always + mem_limit: ${BFF_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${BFF_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + depends_on: + db: + condition: service_healthy + redis: + condition: service_started + environment: + BFF_BASE_URL: http://bff:3000 + EXCEL_PARSER_URL: http://excel_parser:8000 + DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/bff?sslmode=disable&connect_timeout=100" + JWKS_URI: ${APP_JWKS_URI} + MINIO_PDF_BUCKET: bff + MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} + MINIO_ENDPOINT: ${DOMAIN_SCHEME:-https}://cdn-api.${DOMAIN_NAME} + MINIO_PORT: 443 + MINIO_SECURE_CONN: "True" + MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} + OPENAI_API_KEY: ${OPENAI_API_KEY} + PDF_PARSER_BASE_URL: ${DOMAIN_SCHEME:-https}://${DOMAIN_NAME}/pdf-parser + PDF_SPLIT: '50' + REDIS_DB: 8 + REDIS_HOST: redis + REDIS_PORT: 6379 + USER_SERVICE_BASE_URL: http://user_service:3000 + OUTBOUND_BASE_URL: http://inboundjs:3000/outbound + TRANSPORT_SOCKET_BASE_URL: http://transport_socket:3000 + UCI_APIS_BASE_URL: http://uci_apis:3000 + ULCA_API_KEY: ${ULCA_API_KEY} + ULCA_CONFIG_URL: ${ULCA_CONFIG_URL} + ULCA_USER_ID: ${ULCA_USER_ID} + YT_PARSER_URL: http://yt_parser:8000 + CHAT_WITH_DOCS_ORG_OWNER_EMAIL: ${CHAT_WITH_DOCS_ORG_OWNER_EMAIL} + CHAT_WITH_DOCS_ORG_OWNER_PASSWORD: ${CHAT_WITH_DOCS_ORG_OWNER_PASSWORD} + CHAT_WITH_DOCS_PASSWORD_KEY: ${CHAT_WITH_DOCS_PASSWORD_KEY} + CHAT_WITH_DOCS_ORG_ID: ${CHAT_WITH_DOCS_ORG_ID} + REGISTRY_URL: ${REGISTRY_URL} + INBOUNDJS_BASE_URL: http://inboundjs:3000 + WEATHER_VISUAL_CROSSING_BASE_URL: ${WEATHER_VISUAL_CROSSING_BASE_URL} + WEATHER_VISUAL_CROSSING_API_KEY: ${WEATHER_VISUAL_CROSSING_API_KEY} + DOCUMENT_SERVICE_BASE_URL: ${DOMAIN_SCHEME:-https}://${DOMAIN_NAME}/document-service + AZURE_URL: ${AZURE_URL} + USER_SERVICE_ADMIN_API_KEY: ${USER_SERVICE_ADMIN_API_KEY} + USER_SERVICE_URL: ${USER_SERVICE_URL} diff --git a/bhasai/dataset_service/Caddyfile.SDC b/bhasai/dataset_service/Caddyfile.SDC new file mode 100644 index 0000000..5f6b529 --- /dev/null +++ b/bhasai/dataset_service/Caddyfile.SDC @@ -0,0 +1,3 @@ +handle_path /dataset-service* { + reverse_proxy dataset_service:3000 +} diff --git a/bhasai/dataset_service/docker-compose.SDC.yaml b/bhasai/dataset_service/docker-compose.SDC.yaml new file mode 100644 index 0000000..2416049 --- /dev/null +++ b/bhasai/dataset_service/docker-compose.SDC.yaml @@ -0,0 +1,19 @@ +services: + dataset_service: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/dataset-service:${DATASET_SERVICE_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + restart: always + mem_limit: ${DATASET_SERVICE_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${DATASET_SERVICE_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + environment: + DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/dataset_service" + MINIO_ENDPOINT: "${DOMAIN_SCHEME}://cdn-api.${DOMAIN_NAME}" + MINIO_SECURE_CONN: "True" + MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} + MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} + MINIO_BUCKET: dataset-service + OPENAI_API_KEY: ${OPENAI_API_KEY} + REDIS_HOST: redis + REDIS_PORT: 6379 + REDIS_DB: 9 + AI_TOOLS_EMBEDDING_BASE_URL: ${DOMAIN_SCHEME:-https}://ai-tools.${DOMAIN_NAME} + AI_TOOLS_BATCH_SIZE: ${AI_TOOLS_EMBEDDING_BATCH_SIZE} diff --git a/bhasai/deployer/Caddyfile.SDC b/bhasai/deployer/Caddyfile.SDC new file mode 100644 index 0000000..2f3bd9c --- /dev/null +++ b/bhasai/deployer/Caddyfile.SDC @@ -0,0 +1,3 @@ +handle_path /deployer* { + reverse_proxy deployer:3000 +} diff --git a/bhasai/deployer/docker-compose.SDC.yaml b/bhasai/deployer/docker-compose.SDC.yaml new file mode 100644 index 0000000..dc7b01d --- /dev/null +++ b/bhasai/deployer/docker-compose.SDC.yaml @@ -0,0 +1,20 @@ +services: + deployer: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/deployer:${DEPLOYER_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + restart: always + mem_limit: ${DEPLOYER_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${DEPLOYER_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + environment: + DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/deployer?sslmode=disable&connect_timeout=100" + VERCEL_TOKEN: ${DEPLOYER_VERCEL_TOKEN} + PWA_REPO: ${DEPLOYER_PWA_REPO} + APP_ENV: ${DEPLOYER_APP_ENV} + SELF_BASE_URL: ${DOMAIN_SCHEME:-https}://${DOMAIN_NAME}/deployer + MINIO_BUCKETNAME: deployer + STORAGE_ENDPOINT: ${DOMAIN_SCHEME:-https}://cdn-api.${DOMAIN_NAME} + STORAGE_PORT: 443 + STORAGE_USE_SSL: 'true' + STORAGE_ACCESS_KEY: ${MINIO_ACCESS_KEY} + STORAGE_SECRET_KEY: ${MINIO_SECRET_KEY} + STORAGE_MODE: "minio" + AUTH_SERVICE_URL: http://auth_service:3000 \ No newline at end of file diff --git a/bhasai/docker-compose.SDC.yaml b/bhasai/docker-compose.SDC.yaml new file mode 100644 index 0000000..7e02fd5 --- /dev/null +++ b/bhasai/docker-compose.SDC.yaml @@ -0,0 +1,19 @@ +include: + - ./admin/docker-compose.SDC.yaml + - ./ai_tools_proxy/docker-compose.SDC.yaml + - ./auth_service/docker-compose.SDC.yaml + - ./autotune/docker-compose.SDC.yaml + - ./bff/docker-compose.SDC.yaml + - ./dataset_service/docker-compose.SDC.yaml + - ./deployer/docker-compose.SDC.yaml + - ./document_service/docker-compose.SDC.yaml + - ./inboundjs/docker-compose.SDC.yaml + - ./nodehub/docker-compose.SDC.yaml + - ./orchestrator/docker-compose.SDC.yaml + - ./pdf_parser/docker-compose.SDC.yaml + - ./telemetry/docker-compose.SDC.yaml + - ./transport_socket/docker-compose.SDC.yaml + - ./uci_apis/docker-compose.SDC.yaml + - ./user_management_service/docker-compose.SDC.yaml + - ./vault/docker-compose.yaml + - ./yt_parser/docker-compose.SDC.yaml diff --git a/bhasai/document_service/Caddyfile.SDC b/bhasai/document_service/Caddyfile.SDC new file mode 100644 index 0000000..71c6ca0 --- /dev/null +++ b/bhasai/document_service/Caddyfile.SDC @@ -0,0 +1,3 @@ +handle_path /document-service* { + reverse_proxy document_service:3000 +} diff --git a/bhasai/document_service/docker-compose.SDC.yaml b/bhasai/document_service/docker-compose.SDC.yaml new file mode 100644 index 0000000..53700ba --- /dev/null +++ b/bhasai/document_service/docker-compose.SDC.yaml @@ -0,0 +1,22 @@ +services: + document_service: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/document-service:${DOCUMENT_SERVICE_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + restart: always + mem_limit: ${DOCUMENT_SERVICE_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${DOCUMENT_SERVICE_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + environment: + DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/document_service" + MINIO_ENDPOINT: "minio" + MINIO_PORT: "9000" + MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} + MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} + MINIO_BUCKET: document-service + MINIO_SECURE_CONN: "False" + OPENAI_API_KEY: ${OPENAI_API_KEY} + PDF_PARSER_URL: http://pdf_parser:8000 + YT_PARSER_URL: http://yt_parser:8000 + REDIS_HOST: redis + REDIS_PORT: 6379 + REDIS_DB: 4 + AI_TOOLS_EMBEDDING_BASE_URL: ${DOMAIN_SCHEME:-https}://ai-tools.${DOMAIN_NAME} + AI_TOOLS_BATCH_SIZE: ${AI_TOOLS_EMBEDDING_BATCH_SIZE} diff --git a/bhasai/inboundjs/Caddyfile.SDC b/bhasai/inboundjs/Caddyfile.SDC new file mode 100644 index 0000000..47b576e --- /dev/null +++ b/bhasai/inboundjs/Caddyfile.SDC @@ -0,0 +1,3 @@ +handle_path /inboundjs* { + reverse_proxy inboundjs:3000 +} diff --git a/bhasai/inboundjs/docker-compose.SDC.yaml b/bhasai/inboundjs/docker-compose.SDC.yaml new file mode 100644 index 0000000..56ab024 --- /dev/null +++ b/bhasai/inboundjs/docker-compose.SDC.yaml @@ -0,0 +1,13 @@ +services: + inboundjs: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/inbound-js:${INBOUNDJS_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + mem_limit: ${INBOUNDJS_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${INBOUNDJS_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + restart: always + environment: + DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/inboundjs" + ORCHESTRATOR_API_ENDPOINT: "http://orchestrator:3000" + UCI_API_BASE_URL: http://uci-apis:3000 + UCI_API_TOKEN: ${UCI_ADMIN_TOKEN} + TELEMETRY_SERVICE_HOST: http://telemetry:3000 + ENVIRONMENT: ${env} \ No newline at end of file diff --git a/bhasai/nodehub/Caddyfile.SDC b/bhasai/nodehub/Caddyfile.SDC new file mode 100644 index 0000000..63479ea --- /dev/null +++ b/bhasai/nodehub/Caddyfile.SDC @@ -0,0 +1,3 @@ +handle_path /nodehub* { + reverse_proxy nodehub:3000 +} diff --git a/bhasai/nodehub/docker-compose.SDC.yaml b/bhasai/nodehub/docker-compose.SDC.yaml new file mode 100644 index 0000000..413697a --- /dev/null +++ b/bhasai/nodehub/docker-compose.SDC.yaml @@ -0,0 +1,11 @@ +services: + nodehub: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/nodehub:${NODEHUB_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + mem_limit: ${NODEHUB_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${NODEHUB_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + restart: always + environment: + DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/nodehub" + AUTH_SERVICE_URL: "http://auth_service:3000" + REGISTRY_BASE_URL: ${REGISTRY_BASE_URL} + ENVIRONMENT: ${env} \ No newline at end of file diff --git a/bhasai/orchestrator/docker-compose.SDC.yaml b/bhasai/orchestrator/docker-compose.SDC.yaml new file mode 100644 index 0000000..cd02821 --- /dev/null +++ b/bhasai/orchestrator/docker-compose.SDC.yaml @@ -0,0 +1,9 @@ +services: + orchestrator: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/orchestrator:${ORCHESTRATOR_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + mem_limit: ${ORCHESTRATOR_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${ORCHESTRATOR_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + restart: always + environment: + OUTBOUND_SERVICE_BASE_URL: "http://inboundjs:3000/outbound" + ENVIRONMENT: ${env} diff --git a/bhasai/pdf_parser/Caddyfile.SDC b/bhasai/pdf_parser/Caddyfile.SDC new file mode 100644 index 0000000..71aae38 --- /dev/null +++ b/bhasai/pdf_parser/Caddyfile.SDC @@ -0,0 +1,3 @@ +handle_path /pdf-parser* { + reverse_proxy pdf_parser:8000 +} diff --git a/bhasai/pdf_parser/docker-compose.SDC.yaml b/bhasai/pdf_parser/docker-compose.SDC.yaml new file mode 100644 index 0000000..f82e4ba --- /dev/null +++ b/bhasai/pdf_parser/docker-compose.SDC.yaml @@ -0,0 +1,82 @@ +services: + pdf_parser: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/pdf-parser:${PDF_PARSER_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + mem_limit: ${PDF_PARSER_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${PDF_PARSER_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + command: bash -c "./setup.sh && python manage.py runserver 0.0.0.0:8000" + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [gpu] + restart: always + environment: + OPENAI_API_KEY: ${OPENAI_API_KEY} + BHASHINI_API_KEY: ${BHASHINI_API_KEY} + WEBHOOK_URL: http://bff:3000/document/processed-webhook + FASTAPI_HOST: http://pdf_parser:8000 + TESSDATA_PREFIX: "/usr/share/tesseract-ocr/5/tessdata" + MINIO_BASE_URL: ${DOMAIN_SCHEME:-https}://cdn-api.${DOMAIN_NAME} + MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} + MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} + MINIO_BUCKET: pdf_parser + MINIO_SECURE_CONN: "True" + ENABLE_TRANSLATE: ${ENABLE_TRANSLATE} + ENABLE_SUMMARIZE: ${ENABLE_SUMMARIZE} + ENABLE_IMAGE_PARSING: ${ENABLE_IMAGE_PARSING} + SPLIT: ${SPLIT} + PARALLEL_FACTOR: ${PARALLEL_FACTOR} + REDIS_CONN: redis://redis:6379/0 + TORCH_DEVICE: ${TORCH_DEVICE} + CELERY_BROKER_URL: redis://redis:6379/0 + CELERY_RESULT_BACKEND: redis://redis:6379/0 + DISCORD_WEBHOOK_URL: ${DISCORD_WEBHOOK_URL} + PDF_PARSER_MAX_RETRIES : ${PDF_PARSER_MAX_RETRIES} + PDF_PARSER_SECRET_KEY : ${PDF_PARSER_SECRET_KEY} + AZURE_TRANSLATE_KEY : ${AZURE_TRANSLATE_KEY} + TRANSLATION_SERVICES : ${TRANSLATION_SERVICES} + depends_on: + - redis + + pdf_parser_celery_worker: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/pdf-parser:${PDF_PARSER_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + command: bash -c "./setup.sh && celery -A pdf_parser worker -P solo -l info" + restart: always + mem_limit: ${PDF_PARSER_CELERY_WORKER_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${PDF_PARSER_CELERY_WORKER_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [gpu] + environment: + CELERY_BROKER_URL: redis://redis:6379/0 + CELERY_RESULT_BACKEND: redis://redis:6379/0 + OPENAI_API_KEY: ${OPENAI_API_KEY} + BHASHINI_API_KEY: ${BHASHINI_API_KEY} + WEBHOOK_URL: http://bff:3000/document/processed-webhook + FASTAPI_HOST: http://pdf_parser:8000 + TESSDATA_PREFIX: "/usr/share/tesseract-ocr/5/tessdata" + MINIO_BASE_URL: ${DOMAIN_SCHEME:-https}://cdn-api.${DOMAIN_NAME} + MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} + MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} + MINIO_BUCKET: pdf_parser + MINIO_SECURE_CONN: "True" + ENABLE_TRANSLATE: ${ENABLE_TRANSLATE} + ENABLE_SUMMARIZE: ${ENABLE_SUMMARIZE} + ENABLE_IMAGE_PARSING: ${ENABLE_IMAGE_PARSING} + SPLIT: ${SPLIT} + PARALLEL_FACTOR: ${PARALLEL_FACTOR} + REDIS_CONN: redis://redis:6379/0 + TORCH_DEVICE: ${TORCH_DEVICE} + DISCORD_WEBHOOK_URL: ${DISCORD_WEBHOOK_URL} + PDF_PARSER_MAX_RETRIES : ${PDF_PARSER_MAX_RETRIES} + PDF_PARSER_SECRET_KEY : ${PDF_PARSER_SECRET_KEY} + AZURE_TRANSLATE_KEY : ${AZURE_TRANSLATE_KEY} + TRANSLATION_SERVICES : ${TRANSLATION_SERVICES} + depends_on: + - redis \ No newline at end of file diff --git a/bhasai/telemetry/Caddyfile.SDC b/bhasai/telemetry/Caddyfile.SDC new file mode 100644 index 0000000..dbc64b5 --- /dev/null +++ b/bhasai/telemetry/Caddyfile.SDC @@ -0,0 +1,3 @@ +handle_path /telemetry* { + reverse_proxy telemetry:3000 +} diff --git a/bhasai/telemetry/docker-compose.SDC.yaml b/bhasai/telemetry/docker-compose.SDC.yaml new file mode 100644 index 0000000..0a713a0 --- /dev/null +++ b/bhasai/telemetry/docker-compose.SDC.yaml @@ -0,0 +1,13 @@ +services: + telemetry: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/telemetry:${TELEMETRY_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + restart: always + mem_limit: ${TELEMETRY_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${TELEMETRY_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + environment: + DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/telemetry" + CLICKHOUSE_HOST: http://clickhouse:8123 + CLICKHOUSE_DB: ${CLICKHOUSE_DB} + CLICKHOUSE_USER: ${CLICKHOUSE_USER} + CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD} + AUTH_SERVICE_URL: http://auth_service:3000 diff --git a/bhasai/transport_socket/Caddyfile.SDC b/bhasai/transport_socket/Caddyfile.SDC new file mode 100644 index 0000000..41340bd --- /dev/null +++ b/bhasai/transport_socket/Caddyfile.SDC @@ -0,0 +1,3 @@ +handle_path /transport-socket* { + reverse_proxy transport_socket:3000 +} diff --git a/bhasai/transport_socket/docker-compose.SDC.yaml b/bhasai/transport_socket/docker-compose.SDC.yaml new file mode 100644 index 0000000..31b51eb --- /dev/null +++ b/bhasai/transport_socket/docker-compose.SDC.yaml @@ -0,0 +1,12 @@ +services: + transport_socket: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/transport-socket:${TRANSPORT_SOCKET_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + restart: always + mem_limit: ${TRANSPORT_SOCKET_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${TRANSPORT_SOCKET_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + environment: + JWKS_URI: http://fusionauth:9011/.well-known/jwks.json + REDIS_URL: redis://redis:6379/1 + ADAPTER_URL: http://inboundjs:3000/inbound/bot + DEBUG: ${TRANSPORT_SOCKET_DEBUG} + SERVER_PORT: 3000 \ No newline at end of file diff --git a/bhasai/uci_apis/Caddyfile.SDC b/bhasai/uci_apis/Caddyfile.SDC new file mode 100644 index 0000000..0b50acb --- /dev/null +++ b/bhasai/uci_apis/Caddyfile.SDC @@ -0,0 +1,3 @@ +handle_path /uci_apis* { + reverse_proxy uci_apis:3000 +} diff --git a/bhasai/uci_apis/docker-compose.SDC.yaml b/bhasai/uci_apis/docker-compose.SDC.yaml new file mode 100644 index 0000000..2f1443b --- /dev/null +++ b/bhasai/uci_apis/docker-compose.SDC.yaml @@ -0,0 +1,19 @@ +services: + uci_apis: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/uci-apis:${UCI_APIS_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + restart: always + mem_limit: ${UCI_APIS_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${UCI_APIS_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + depends_on: + db: + condition: service_healthy + environment: + UCI_API_PSQL_DB_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/uci_apis" + REDIS_URL: redis://redis:6379/2 + FUSIONAUTH_URL: http://fusionauth:9011 + FUSIONAUTH_KEY: ${FUSIONAUTH_API_KEY} + ADMIN_TOKEN: ${UCI_ADMIN_TOKEN} + VAULT_ADDR: http://vault:8200 + VAULT_USERNAME: ${VAULT_USERNAME} + VAULT_PASSWORD: ${VAULT_PASSWORD} + AUTH_SERVICE_URL: http://auth_service:3000 \ No newline at end of file diff --git a/bhasai/user_management_service/Caddyfile.SDC b/bhasai/user_management_service/Caddyfile.SDC new file mode 100644 index 0000000..6953d3f --- /dev/null +++ b/bhasai/user_management_service/Caddyfile.SDC @@ -0,0 +1,3 @@ +handle_path /user-management-service* { + reverse_proxy user_management_service:3000 +} diff --git a/bhasai/user_management_service/docker-compose.SDC.yaml b/bhasai/user_management_service/docker-compose.SDC.yaml new file mode 100644 index 0000000..b3e0afa --- /dev/null +++ b/bhasai/user_management_service/docker-compose.SDC.yaml @@ -0,0 +1,11 @@ +services: + user_management_service: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/user-management-service:${USER_MANAGEMENT_SERVICE_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + restart: always + mem_limit: ${USER_MANAGEMENT_SERVICE_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${USER_MANAGEMENT_SERVICE_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + depends_on: + db: + condition: service_healthy + environment: + DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/user_management_service" diff --git a/bhasai/yt_parser/Caddyfile.SDC b/bhasai/yt_parser/Caddyfile.SDC new file mode 100644 index 0000000..6d0de12 --- /dev/null +++ b/bhasai/yt_parser/Caddyfile.SDC @@ -0,0 +1,3 @@ +handle_path /yt-parser* { + reverse_proxy yt_parser:8000 +} diff --git a/bhasai/yt_parser/docker-compose.SDC.yaml b/bhasai/yt_parser/docker-compose.SDC.yaml new file mode 100644 index 0000000..6744591 --- /dev/null +++ b/bhasai/yt_parser/docker-compose.SDC.yaml @@ -0,0 +1,51 @@ +services: + yt_parser: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/yt-parser:${YT_PARSER_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + command: uvicorn main:app --host 0.0.0.0 --port 8000 + restart: always + mem_limit: ${YT_PARSER_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${YT_PARSER_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + environment: + MINIO_BASE_URL: "minio:9000" + MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} + MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} + MINIO_BUCKET: yt_parser + MINIO_SECURE_CONN: "False" + REDIS_CONN: redis://redis:6379/3 + CELERY_BROKER_URL: redis://redis:6379/3 + CELERY_RESULT_BACKEND: redis://redis:6379/3 + BHASHINI_USERID: ${BHASHINI_USERID} + BHASHINI_ULCA_API_KEY: ${BHASHINI_ULCA_API_KEY} + AZURE_SUBSCRIPTION_KEY: ${AZURE_SUBSCRIPTION_KEY} + WEBHOOK_URL: http://bff:3000/document/processed-webhook/youtube + DISCORD_WEBHOOK_URL: ${DISCORD_WEBHOOK_URL} + ASR_SERVICE_PRIORITY: ${ASR_SERVICE_PRIORITY} + YOUTUBE_DATA_API_KEY: ${YOUTUBE_DATA_API_KEY} + depends_on: + - redis + + yt_parser_celery_worker: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/yt-parser:${YT_PARSER_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + command: celery -A worker worker -P gevent --loglevel=info + restart: always + mem_limit: ${YT_PARSER_CELERY_WORKER_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${YT_PARSER_CELERY_WORKER_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + environment: + CELERY_BROKER_URL: redis://redis:6379/3 + CELERY_RESULT_BACKEND: redis://redis:6379/3 + WEBHOOK_URL: http://bff:3000/document/processed-webhook/youtube + MINIO_BASE_URL: "minio:9000" + MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} + MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} + MINIO_BUCKET: yt_parser + MINIO_SECURE_CONN: "False" + REDIS_CONN: redis://redis:6379/3 + BHASHINI_USERID: ${BHASHINI_USERID} + BHASHINI_ULCA_API_KEY: ${BHASHINI_ULCA_API_KEY} + AZURE_SUBSCRIPTION_KEY: ${AZURE_SUBSCRIPTION_KEY} + DISCORD_WEBHOOK_URL: ${DISCORD_WEBHOOK_URL} + ASR_SERVICE_PRIORITY: ${ASR_SERVICE_PRIORITY} + YOUTUBE_DATA_API_KEY: ${YOUTUBE_DATA_API_KEY} + depends_on: + - redis + \ No newline at end of file diff --git a/docker-compose.SDC.yaml b/docker-compose.SDC.yaml new file mode 100644 index 0000000..fad5fb1 --- /dev/null +++ b/docker-compose.SDC.yaml @@ -0,0 +1,24 @@ +#include: +# - ./common/monitoring/docker-compose.SDC.yaml +# - ./common/minio/docker-compose.yaml +# - ./common/environment/docker-compose.yaml +# - ./common/fusionauth/docker-compose.yaml +# - ./common/uptime/docker-compose.yaml +# - ./common/clickhouse/docker-compose.yaml +# - ./common/redis/docker-compose.yaml +# - ./common/db/docker-compose.yaml +# - ./common/registry/docker-compose.yaml + + +services: + caddy: + extends: + file: common/caddy/docker-compose.SDC.yaml + service: caddy +# volumes: +# - ./certs:/certs + + +volumes: + caddy_data: + caddy_config: From ea9c55470773ef998c5d660e7b03bf6c317c2c1c Mon Sep 17 00:00:00 2001 From: Karun Agarwal <113603846+singhalkarun@users.noreply.github.com> Date: Fri, 2 Aug 2024 13:29:24 +0000 Subject: [PATCH 04/28] add SERVE_PATH in admin --- bhasai/admin/docker-compose.SDC.yaml | 1 + bhasai/admin/docker-compose.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/bhasai/admin/docker-compose.SDC.yaml b/bhasai/admin/docker-compose.SDC.yaml index 0ea724a..f6bf3dd 100644 --- a/bhasai/admin/docker-compose.SDC.yaml +++ b/bhasai/admin/docker-compose.SDC.yaml @@ -13,4 +13,5 @@ services: NEXT_PUBLIC_TELEMETRY_API_URL: ${DOMAIN_SCHEME:-https}://${DOMAIN_NAME}/telemetry NEXT_PUBLIC_FLOWISE_URL: ${NEXT_PUBLIC_FLOWISE_URL} NEXT_PUBLIC_AUTOTUNE_API_URL: ${DOMAIN_SCHEME:-https}://${DOMAIN_NAME}/autotune + SERVE_PATH: "/admin" restart: always diff --git a/bhasai/admin/docker-compose.yaml b/bhasai/admin/docker-compose.yaml index 1151d52..0e4732e 100644 --- a/bhasai/admin/docker-compose.yaml +++ b/bhasai/admin/docker-compose.yaml @@ -13,4 +13,5 @@ services: NEXT_PUBLIC_TELEMETRY_API_URL: ${DOMAIN_SCHEME:-https}://telemetry.${DOMAIN_NAME} NEXT_PUBLIC_FLOWISE_URL: ${NEXT_PUBLIC_FLOWISE_URL} NEXT_PUBLIC_AUTOTUNE_API_URL: ${DOMAIN_SCHEME:-https}://autotune.${DOMAIN_NAME} + SERVE_PATH: "" restart: always From 5f22f7aea6742463d4985c7c08feaa1f5a5a20f5 Mon Sep 17 00:00:00 2001 From: Karun Agarwal <113603846+singhalkarun@users.noreply.github.com> Date: Mon, 5 Aug 2024 06:51:11 +0000 Subject: [PATCH 05/28] add flowise in bhasai and onboard it to SDC root files --- bhasai/Caddyfile.SDC | 3 ++- bhasai/docker-compose.SDC.yaml | 1 + bhasai/flowise/Caddyfile | 3 +++ bhasai/flowise/Caddyfile.SDC | 3 +++ bhasai/flowise/docker-compose.SDC.yaml | 10 ++++++++++ bhasai/flowise/docker-compose.yaml | 10 ++++++++++ bhasai/flowise/sample.env | 3 +++ 7 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 bhasai/flowise/Caddyfile create mode 100644 bhasai/flowise/Caddyfile.SDC create mode 100644 bhasai/flowise/docker-compose.SDC.yaml create mode 100644 bhasai/flowise/docker-compose.yaml create mode 100644 bhasai/flowise/sample.env diff --git a/bhasai/Caddyfile.SDC b/bhasai/Caddyfile.SDC index ef1e1f9..14e52ef 100644 --- a/bhasai/Caddyfile.SDC +++ b/bhasai/Caddyfile.SDC @@ -13,4 +13,5 @@ import ./telemetry/Caddyfile.SDC import ./transport_socket/Caddyfile.SDC import ./uci_apis/Caddyfile.SDC import ./user_management_service/Caddyfile.SDC -import ./yt_parser/Caddyfile.SDC \ No newline at end of file +import ./yt_parser/Caddyfile.SDC +import ./flowise/Caddyfile.SDC \ No newline at end of file diff --git a/bhasai/docker-compose.SDC.yaml b/bhasai/docker-compose.SDC.yaml index 7e02fd5..cc49852 100644 --- a/bhasai/docker-compose.SDC.yaml +++ b/bhasai/docker-compose.SDC.yaml @@ -17,3 +17,4 @@ include: - ./user_management_service/docker-compose.SDC.yaml - ./vault/docker-compose.yaml - ./yt_parser/docker-compose.SDC.yaml + - ./flowise/docker-compose.SDC.yaml \ No newline at end of file diff --git a/bhasai/flowise/Caddyfile b/bhasai/flowise/Caddyfile new file mode 100644 index 0000000..8c7269e --- /dev/null +++ b/bhasai/flowise/Caddyfile @@ -0,0 +1,3 @@ +{$DOMAIN_NAME}://flowise.{$DOMAIN_NAME} { + reverse_proxy flowise:3000 +} \ No newline at end of file diff --git a/bhasai/flowise/Caddyfile.SDC b/bhasai/flowise/Caddyfile.SDC new file mode 100644 index 0000000..ee05ab9 --- /dev/null +++ b/bhasai/flowise/Caddyfile.SDC @@ -0,0 +1,3 @@ +handle /flowise* { + reverse_proxy flowise:3000 +} diff --git a/bhasai/flowise/docker-compose.SDC.yaml b/bhasai/flowise/docker-compose.SDC.yaml new file mode 100644 index 0000000..96facb1 --- /dev/null +++ b/bhasai/flowise/docker-compose.SDC.yaml @@ -0,0 +1,10 @@ +services: + admin: + build: + context: https://${GIT_USERNAME}:${GIT_PASSWORD}@github.com/${org}/flowise.git#${APP_GITHUB_BRANCH:-${DEFAULT_GITHUB_BRANCH:?DEFAULT_GITHUB_BRANCH is not set}} + args: + REACT_APP_UCI_API: ${DOMAIN_SCHEME:-https}://${DOMAIN_NAME}/uci-apis + REACT_APP_SOCKET_URL: ${DOMAIN_SCHEME:-https}://${DOMAIN_NAME}/transport-socket + REACT_APP_REGISTRY_BASE_URL: ${DOMAIN_SCHEME:-https}://${DOMAIN_NAME}/nodehub + SERVE_PATH: "/flowise" + restart: always diff --git a/bhasai/flowise/docker-compose.yaml b/bhasai/flowise/docker-compose.yaml new file mode 100644 index 0000000..b5dd296 --- /dev/null +++ b/bhasai/flowise/docker-compose.yaml @@ -0,0 +1,10 @@ +services: + admin: + build: + context: https://${GIT_USERNAME}:${GIT_PASSWORD}@github.com/${org}/flowise.git#${APP_GITHUB_BRANCH:-${DEFAULT_GITHUB_BRANCH:?DEFAULT_GITHUB_BRANCH is not set}} + args: + REACT_APP_UCI_API: ${DOMAIN_SCHEME:-https}://uci-apis.${DOMAIN_NAME} + REACT_APP_SOCKET_URL: ${DOMAIN_SCHEME:-https}://transport-socket.${DOMAIN_NAME} + REACT_APP_REGISTRY_BASE_URL: ${DOMAIN_SCHEME:-https}://nodehub.${DOMAIN_NAME} + SERVE_PATH: "" + restart: always diff --git a/bhasai/flowise/sample.env b/bhasai/flowise/sample.env new file mode 100644 index 0000000..2389c05 --- /dev/null +++ b/bhasai/flowise/sample.env @@ -0,0 +1,3 @@ +#flowise +DOMAIN_NAME= +DOMAIN_SCHEME= \ No newline at end of file From 0831a52ea303a9eb3498454fa80d75ec69c0a772 Mon Sep 17 00:00:00 2001 From: Karun Agarwal <113603846+singhalkarun@users.noreply.github.com> Date: Mon, 5 Aug 2024 13:05:32 +0000 Subject: [PATCH 06/28] fix typo in flowise branch tag --- bhasai/flowise/docker-compose.SDC.yaml | 2 +- bhasai/flowise/docker-compose.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bhasai/flowise/docker-compose.SDC.yaml b/bhasai/flowise/docker-compose.SDC.yaml index 96facb1..1d2319d 100644 --- a/bhasai/flowise/docker-compose.SDC.yaml +++ b/bhasai/flowise/docker-compose.SDC.yaml @@ -1,7 +1,7 @@ services: admin: build: - context: https://${GIT_USERNAME}:${GIT_PASSWORD}@github.com/${org}/flowise.git#${APP_GITHUB_BRANCH:-${DEFAULT_GITHUB_BRANCH:?DEFAULT_GITHUB_BRANCH is not set}} + context: https://${GIT_USERNAME}:${GIT_PASSWORD}@github.com/${org}/flowise.git#${FLOWISE_GITHUB_BRANCH:-${DEFAULT_GITHUB_BRANCH:?DEFAULT_GITHUB_BRANCH is not set}} args: REACT_APP_UCI_API: ${DOMAIN_SCHEME:-https}://${DOMAIN_NAME}/uci-apis REACT_APP_SOCKET_URL: ${DOMAIN_SCHEME:-https}://${DOMAIN_NAME}/transport-socket diff --git a/bhasai/flowise/docker-compose.yaml b/bhasai/flowise/docker-compose.yaml index b5dd296..d30f01b 100644 --- a/bhasai/flowise/docker-compose.yaml +++ b/bhasai/flowise/docker-compose.yaml @@ -1,7 +1,7 @@ services: admin: build: - context: https://${GIT_USERNAME}:${GIT_PASSWORD}@github.com/${org}/flowise.git#${APP_GITHUB_BRANCH:-${DEFAULT_GITHUB_BRANCH:?DEFAULT_GITHUB_BRANCH is not set}} + context: https://${GIT_USERNAME}:${GIT_PASSWORD}@github.com/${org}/flowise.git#${FLOWISE_GITHUB_BRANCH:-${DEFAULT_GITHUB_BRANCH:?DEFAULT_GITHUB_BRANCH is not set}} args: REACT_APP_UCI_API: ${DOMAIN_SCHEME:-https}://uci-apis.${DOMAIN_NAME} REACT_APP_SOCKET_URL: ${DOMAIN_SCHEME:-https}://transport-socket.${DOMAIN_NAME} From 19c2a01fe9fa6da68130a987e13558ea006704ee Mon Sep 17 00:00:00 2001 From: Srijan Srivastava Date: Tue, 6 Aug 2024 10:02:35 +0530 Subject: [PATCH 07/28] added app for SDC (#27) --- bhasai/Caddyfile.SDC | 3 ++- bhasai/app/Caddyfile | 3 +++ bhasai/app/Caddyfile.SDC | 3 +++ bhasai/app/docker-compose.SDC.yaml | 27 +++++++++++++++++++++++++++ bhasai/app/docker-compose.yaml | 27 +++++++++++++++++++++++++++ bhasai/app/sample.env | 12 ++++++++++++ bhasai/docker-compose.SDC.yaml | 3 ++- 7 files changed, 76 insertions(+), 2 deletions(-) create mode 100644 bhasai/app/Caddyfile create mode 100644 bhasai/app/Caddyfile.SDC create mode 100644 bhasai/app/docker-compose.SDC.yaml create mode 100644 bhasai/app/docker-compose.yaml create mode 100644 bhasai/app/sample.env diff --git a/bhasai/Caddyfile.SDC b/bhasai/Caddyfile.SDC index 14e52ef..1d69a40 100644 --- a/bhasai/Caddyfile.SDC +++ b/bhasai/Caddyfile.SDC @@ -14,4 +14,5 @@ import ./transport_socket/Caddyfile.SDC import ./uci_apis/Caddyfile.SDC import ./user_management_service/Caddyfile.SDC import ./yt_parser/Caddyfile.SDC -import ./flowise/Caddyfile.SDC \ No newline at end of file +import ./flowise/Caddyfile.SDC +import ./app/Caddyfile.SDC \ No newline at end of file diff --git a/bhasai/app/Caddyfile b/bhasai/app/Caddyfile new file mode 100644 index 0000000..bdb3acd --- /dev/null +++ b/bhasai/app/Caddyfile @@ -0,0 +1,3 @@ +{$DOMAIN_NAME}://app.{$DOMAIN_NAME} { + reverse_proxy app:3000 +} \ No newline at end of file diff --git a/bhasai/app/Caddyfile.SDC b/bhasai/app/Caddyfile.SDC new file mode 100644 index 0000000..7628748 --- /dev/null +++ b/bhasai/app/Caddyfile.SDC @@ -0,0 +1,3 @@ +handle /* { + reverse_proxy app:3000 +} diff --git a/bhasai/app/docker-compose.SDC.yaml b/bhasai/app/docker-compose.SDC.yaml new file mode 100644 index 0000000..de51280 --- /dev/null +++ b/bhasai/app/docker-compose.SDC.yaml @@ -0,0 +1,27 @@ +services: + app: + build: + context: https://${GIT_USERNAME}:${GIT_PASSWORD}@github.com/${org}/kisai-bot.git#${APP_GITHUB_BRANCH:-${DEFAULT_GITHUB_BRANCH:?DEFAULT_GITHUB_BRANCH is not set}} + args: + NEXT_PUBLIC_BOT_ID: ${NEXT_PUBLIC_BOT_ID} + NEXT_PUBLIC_CONFIG_BASE_URL: ${DOMAIN_SCHEME:-https}://${DOMAIN_NAME}/deployer/deployment/config/${NEXT_PUBLIC_BOT_ID} + NEXT_PUBLIC_BOT_NAME: ${NEXT_PUBLIC_BOT_NAME} + NEXT_PUBLIC_SOCKET_URL: ${DOMAIN_SCHEME:-https}://${DOMAIN_NAME} + NEXT_PUBLIC_TELEMETRY_API: ${DOMAIN_SCHEME:-https}://${DOMAIN_NAME}/telemetry + NEXT_PUBLIC_AI_TOOLS_API: ${DOMAIN_SCHEME:-https}://${DOMAIN_NAME}/ai-tools-proxy + NEXT_PUBLIC_BFF_API_URL: ${DOMAIN_SCHEME:-https}://${DOMAIN_NAME}/bff + NEXT_PUBLIC_USER_SERVICE_APP_ID: ${NEXT_PUBLIC_USER_SERVICE_APP_ID} + NEXT_PUBLIC_JWKS_URI: 'http://fusionauth:9011/.well-known/jwks.json' + NEXT_PUBLIC_USER_SERVICE_URL: ${DOMAIN_SCHEME:-https}://${DOMAIN_NAME}/user + NEXT_PUBLIC_ORG_ID: ${NEXT_PUBLIC_ORG_ID} + NEXT_PUBLIC_DEBUG: ${NEXT_PUBLIC_DEBUG} + NEXT_PUBLIC_SOCKET_PATH: '/transport-socket/socket.io' + NEXT_PUBLIC_GOOGLE_KEY: ${NEXT_PUBLIC_GOOGLE_KEY} + FUSIONAUTH_URL: http://fusionauth:9011/ + FUSIONAUTH_KEY: ${FUSIONAUTH_KEY} + NEXT_PUBLIC_DATASET_URL: ${DOMAIN_SCHEME:-https}://${DOMAIN_NAME}/dataset-service + NEXT_PUBLIC_ENTITY_DATASET_ID: ${NEXT_PUBLIC_ENTITY_DATASET_ID} + NEXT_PUBLIC_CLARITY_PROJECT_ID: ${NEXT_PUBLIC_CLARITY_PROJECT_ID} + NEXT_PUBLIC_SHOW_ONBOARDING: ${NEXT_PUBLIC_SHOW_ONBOARDING} + NEXT_PUBLIC_WEATHER_API: ${NEXT_PUBLIC_WEATHER_API} + \ No newline at end of file diff --git a/bhasai/app/docker-compose.yaml b/bhasai/app/docker-compose.yaml new file mode 100644 index 0000000..cba3ccf --- /dev/null +++ b/bhasai/app/docker-compose.yaml @@ -0,0 +1,27 @@ +services: + app: + build: + context: https://${GIT_USERNAME}:${GIT_PASSWORD}@github.com/${org}/kisai-bot.git#${APP_GITHUB_BRANCH:-${DEFAULT_GITHUB_BRANCH:?DEFAULT_GITHUB_BRANCH is not set}} + args: + NEXT_PUBLIC_BOT_ID: ${NEXT_PUBLIC_BOT_ID} + NEXT_PUBLIC_CONFIG_BASE_URL: ${DOMAIN_SCHEME:-https}://deployer.${DOMAIN_NAME}/deployment/config/${NEXT_PUBLIC_BOT_ID} + NEXT_PUBLIC_BOT_NAME: ${NEXT_PUBLIC_BOT_NAME} + NEXT_PUBLIC_SOCKET_URL: ${DOMAIN_SCHEME:-https}://transport-socket.${DOMAIN_NAME} + NEXT_PUBLIC_TELEMETRY_API: ${DOMAIN_SCHEME:-https}://telemetry.${DOMAIN_NAME} + NEXT_PUBLIC_AI_TOOLS_API: ${DOMAIN_SCHEME:-https}://ai-tools-proxy.${DOMAIN_NAME} + NEXT_PUBLIC_BFF_API_URL: ${DOMAIN_SCHEME:-https}://bff.${DOMAIN_NAME} + NEXT_PUBLIC_USER_SERVICE_APP_ID: ${NEXT_PUBLIC_USER_SERVICE_APP_ID} + NEXT_PUBLIC_JWKS_URI: 'http://fusionauth:9011/.well-known/jwks.json' + NEXT_PUBLIC_USER_SERVICE_URL: ${DOMAIN_SCHEME:-https}://user.${DOMAIN_NAME} + NEXT_PUBLIC_ORG_ID: ${NEXT_PUBLIC_ORG_ID} + NEXT_PUBLIC_DEBUG: ${NEXT_PUBLIC_DEBUG} + NEXT_PUBLIC_SOCKET_PATH: '' + NEXT_PUBLIC_GOOGLE_KEY: ${NEXT_PUBLIC_GOOGLE_KEY} + FUSIONAUTH_URL: http://fusionauth:9011/ + FUSIONAUTH_KEY: ${FUSIONAUTH_KEY} + NEXT_PUBLIC_DATASET_URL: ${DOMAIN_SCHEME:-https}://dataset-service.${DOMAIN_NAME} + NEXT_PUBLIC_ENTITY_DATASET_ID: ${NEXT_PUBLIC_ENTITY_DATASET_ID} + NEXT_PUBLIC_CLARITY_PROJECT_ID: ${NEXT_PUBLIC_CLARITY_PROJECT_ID} + NEXT_PUBLIC_SHOW_ONBOARDING: ${NEXT_PUBLIC_SHOW_ONBOARDING} + NEXT_PUBLIC_WEATHER_API: ${NEXT_PUBLIC_WEATHER_API} + \ No newline at end of file diff --git a/bhasai/app/sample.env b/bhasai/app/sample.env new file mode 100644 index 0000000..6d93593 --- /dev/null +++ b/bhasai/app/sample.env @@ -0,0 +1,12 @@ +#app +NEXT_PUBLIC_BOT_ID= +NEXT_PUBLIC_ORG_ID= +NEXT_PUBLIC_GOOGLE_KEY= +NEXT_PUBLIC_BOT_NAME= +FUSIONAUTH_KEY= +NEXT_PUBLIC_ENTITY_DATASET_ID= +NEXT_PUBLIC_CLARITY_PROJECT_ID= +NEXT_PUBLIC_USER_SERVICE_APP_ID= +NEXT_PUBLIC_DEBUG= +NEXT_PUBLIC_SHOW_ONBOARDING= +NEXT_PUBLIC_WEATHER_API= \ No newline at end of file diff --git a/bhasai/docker-compose.SDC.yaml b/bhasai/docker-compose.SDC.yaml index cc49852..b0d1b88 100644 --- a/bhasai/docker-compose.SDC.yaml +++ b/bhasai/docker-compose.SDC.yaml @@ -17,4 +17,5 @@ include: - ./user_management_service/docker-compose.SDC.yaml - ./vault/docker-compose.yaml - ./yt_parser/docker-compose.SDC.yaml - - ./flowise/docker-compose.SDC.yaml \ No newline at end of file + - ./flowise/docker-compose.SDC.yaml + - ./app/docker-compose.SDC.yaml \ No newline at end of file From c658a4484f0818c032438ce5c29226832ce16ca5 Mon Sep 17 00:00:00 2001 From: Karun Agarwal <113603846+singhalkarun@users.noreply.github.com> Date: Tue, 6 Aug 2024 12:08:31 +0530 Subject: [PATCH 08/28] added user_service in bhasai and fixed endpoint of user service in app (#29) --- bhasai/Caddyfile.SDC | 3 ++- bhasai/app/docker-compose.SDC.yaml | 2 +- bhasai/app/docker-compose.yaml | 2 +- bhasai/docker-compose.SDC.yaml | 3 ++- bhasai/user_service/Caddyfile | 3 +++ bhasai/user_service/Caddyfile.SDC | 3 +++ bhasai/user_service/docker-compose.SDC.yaml | 20 ++++++++++++++++++++ bhasai/user_service/docker-compose.yaml | 20 ++++++++++++++++++++ bhasai/user_service/sample.env | 14 ++++++++++++++ 9 files changed, 66 insertions(+), 4 deletions(-) create mode 100644 bhasai/user_service/Caddyfile create mode 100644 bhasai/user_service/Caddyfile.SDC create mode 100644 bhasai/user_service/docker-compose.SDC.yaml create mode 100644 bhasai/user_service/docker-compose.yaml create mode 100644 bhasai/user_service/sample.env diff --git a/bhasai/Caddyfile.SDC b/bhasai/Caddyfile.SDC index 1d69a40..928a37e 100644 --- a/bhasai/Caddyfile.SDC +++ b/bhasai/Caddyfile.SDC @@ -15,4 +15,5 @@ import ./uci_apis/Caddyfile.SDC import ./user_management_service/Caddyfile.SDC import ./yt_parser/Caddyfile.SDC import ./flowise/Caddyfile.SDC -import ./app/Caddyfile.SDC \ No newline at end of file +import ./app/Caddyfile.SDC +import ./user_service/Caddyfile.SDC \ No newline at end of file diff --git a/bhasai/app/docker-compose.SDC.yaml b/bhasai/app/docker-compose.SDC.yaml index de51280..c3fd649 100644 --- a/bhasai/app/docker-compose.SDC.yaml +++ b/bhasai/app/docker-compose.SDC.yaml @@ -12,7 +12,7 @@ services: NEXT_PUBLIC_BFF_API_URL: ${DOMAIN_SCHEME:-https}://${DOMAIN_NAME}/bff NEXT_PUBLIC_USER_SERVICE_APP_ID: ${NEXT_PUBLIC_USER_SERVICE_APP_ID} NEXT_PUBLIC_JWKS_URI: 'http://fusionauth:9011/.well-known/jwks.json' - NEXT_PUBLIC_USER_SERVICE_URL: ${DOMAIN_SCHEME:-https}://${DOMAIN_NAME}/user + NEXT_PUBLIC_USER_SERVICE_URL: ${DOMAIN_SCHEME:-https}://${DOMAIN_NAME}/user-service NEXT_PUBLIC_ORG_ID: ${NEXT_PUBLIC_ORG_ID} NEXT_PUBLIC_DEBUG: ${NEXT_PUBLIC_DEBUG} NEXT_PUBLIC_SOCKET_PATH: '/transport-socket/socket.io' diff --git a/bhasai/app/docker-compose.yaml b/bhasai/app/docker-compose.yaml index cba3ccf..ae03abd 100644 --- a/bhasai/app/docker-compose.yaml +++ b/bhasai/app/docker-compose.yaml @@ -12,7 +12,7 @@ services: NEXT_PUBLIC_BFF_API_URL: ${DOMAIN_SCHEME:-https}://bff.${DOMAIN_NAME} NEXT_PUBLIC_USER_SERVICE_APP_ID: ${NEXT_PUBLIC_USER_SERVICE_APP_ID} NEXT_PUBLIC_JWKS_URI: 'http://fusionauth:9011/.well-known/jwks.json' - NEXT_PUBLIC_USER_SERVICE_URL: ${DOMAIN_SCHEME:-https}://user.${DOMAIN_NAME} + NEXT_PUBLIC_USER_SERVICE_URL: ${DOMAIN_SCHEME:-https}://user-service.${DOMAIN_NAME} NEXT_PUBLIC_ORG_ID: ${NEXT_PUBLIC_ORG_ID} NEXT_PUBLIC_DEBUG: ${NEXT_PUBLIC_DEBUG} NEXT_PUBLIC_SOCKET_PATH: '' diff --git a/bhasai/docker-compose.SDC.yaml b/bhasai/docker-compose.SDC.yaml index b0d1b88..af9c63a 100644 --- a/bhasai/docker-compose.SDC.yaml +++ b/bhasai/docker-compose.SDC.yaml @@ -18,4 +18,5 @@ include: - ./vault/docker-compose.yaml - ./yt_parser/docker-compose.SDC.yaml - ./flowise/docker-compose.SDC.yaml - - ./app/docker-compose.SDC.yaml \ No newline at end of file + - ./app/docker-compose.SDC.yaml + - ./user_service/docker-compose.SDC.yaml \ No newline at end of file diff --git a/bhasai/user_service/Caddyfile b/bhasai/user_service/Caddyfile new file mode 100644 index 0000000..06d71c6 --- /dev/null +++ b/bhasai/user_service/Caddyfile @@ -0,0 +1,3 @@ +{$DOMAIN_SCHEME}://user-service.{$DOMAIN_NAME} { + reverse_proxy user_service:3000 +} \ No newline at end of file diff --git a/bhasai/user_service/Caddyfile.SDC b/bhasai/user_service/Caddyfile.SDC new file mode 100644 index 0000000..ea97fb7 --- /dev/null +++ b/bhasai/user_service/Caddyfile.SDC @@ -0,0 +1,3 @@ +handle_path /user-service* { + reverse_proxy user_service:3000 +} diff --git a/bhasai/user_service/docker-compose.SDC.yaml b/bhasai/user_service/docker-compose.SDC.yaml new file mode 100644 index 0000000..94263f8 --- /dev/null +++ b/bhasai/user_service/docker-compose.SDC.yaml @@ -0,0 +1,20 @@ +services: + user_service: + image: samagragovernance/esamwad-user-service:latest + restart: always + mem_limit: ${USER_SERVICE_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${USER_SERVICE_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + environment: + - ADMIN_API_KEY=${ADMIN_API_KEY} + - ALLOW_DEFAULT_OTP=${ALLOW_DEFAULT_OTP:-false} + - DEFAULT_OTP=${DEFAULT_OTP:-1234} + - DEFAULT_OTP_USERS=${DEFAULT_OTP_USERS} + - DEFAULT_USER_PASSWORD=${DEFAULT_USER_PASSWORD} + - DEFAULT_USER_PASSWORD="http://fusionauth:9011" + - FUSIONAUTH_OLD_BASE_URL"="http://fusionauth:9011" + - GUPSHUP_BASEURL="http://enterprise.smsgupshup.com/GatewayAPI/rest" + - GUPSHUP_OTP_TEMPLATE=${GUPSHUP_OTP_TEMPLATE} + - GUPSHUP_PASSWORD=${GUPSHUP_PASSWORD} + - GUPSHUP_USERNAME=${GUPSHUP_USERNAME} + - OTP_EXPIRY=${OTP_EXPIRY:-30} + - APP_${FUSIONAUTH_APP_ID}="{"host":"http://fusionauth:9011","salt":"${FUSIONAUTH_SALT}","apiKey":"${FUSIONAUTH_API_KEY}","encryption":{"enabled":false}}" diff --git a/bhasai/user_service/docker-compose.yaml b/bhasai/user_service/docker-compose.yaml new file mode 100644 index 0000000..94263f8 --- /dev/null +++ b/bhasai/user_service/docker-compose.yaml @@ -0,0 +1,20 @@ +services: + user_service: + image: samagragovernance/esamwad-user-service:latest + restart: always + mem_limit: ${USER_SERVICE_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${USER_SERVICE_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + environment: + - ADMIN_API_KEY=${ADMIN_API_KEY} + - ALLOW_DEFAULT_OTP=${ALLOW_DEFAULT_OTP:-false} + - DEFAULT_OTP=${DEFAULT_OTP:-1234} + - DEFAULT_OTP_USERS=${DEFAULT_OTP_USERS} + - DEFAULT_USER_PASSWORD=${DEFAULT_USER_PASSWORD} + - DEFAULT_USER_PASSWORD="http://fusionauth:9011" + - FUSIONAUTH_OLD_BASE_URL"="http://fusionauth:9011" + - GUPSHUP_BASEURL="http://enterprise.smsgupshup.com/GatewayAPI/rest" + - GUPSHUP_OTP_TEMPLATE=${GUPSHUP_OTP_TEMPLATE} + - GUPSHUP_PASSWORD=${GUPSHUP_PASSWORD} + - GUPSHUP_USERNAME=${GUPSHUP_USERNAME} + - OTP_EXPIRY=${OTP_EXPIRY:-30} + - APP_${FUSIONAUTH_APP_ID}="{"host":"http://fusionauth:9011","salt":"${FUSIONAUTH_SALT}","apiKey":"${FUSIONAUTH_API_KEY}","encryption":{"enabled":false}}" diff --git a/bhasai/user_service/sample.env b/bhasai/user_service/sample.env new file mode 100644 index 0000000..a4922af --- /dev/null +++ b/bhasai/user_service/sample.env @@ -0,0 +1,14 @@ +ADMIN_API_KEY= +ALLOW_DEFAULT_OTP= +DEFAULT_OTP= +DEFAULT_OTP_USERS= +DEFAULT_USER_PASSWORD= +GUPSHUP_OTP_TEMPLATE= +GUPSHUP_PASSWORD= +GUPSHUP_USERNAME= +OTP_EXPIRY= +#fusionauth application id converted to snake case +FUSIONAUTH_APP_ID= +#random string +FUSIONAUTH_SALT= +FUSIONAUTH_API_KEY= \ No newline at end of file From c92cb7063de3dfc2cb22e60d7886a8fbdcfce75b Mon Sep 17 00:00:00 2001 From: Karun Agarwal Date: Tue, 6 Aug 2024 12:14:31 +0530 Subject: [PATCH 09/28] enable necessary services for SDC in docker-compose.SDC.yaml --- docker-compose.SDC.yaml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/docker-compose.SDC.yaml b/docker-compose.SDC.yaml index fad5fb1..c27353b 100644 --- a/docker-compose.SDC.yaml +++ b/docker-compose.SDC.yaml @@ -1,13 +1,14 @@ -#include: -# - ./common/monitoring/docker-compose.SDC.yaml -# - ./common/minio/docker-compose.yaml +include: + - ./common/monitoring/docker-compose.SDC.yaml + - ./common/minio/docker-compose.yaml # - ./common/environment/docker-compose.yaml -# - ./common/fusionauth/docker-compose.yaml -# - ./common/uptime/docker-compose.yaml -# - ./common/clickhouse/docker-compose.yaml -# - ./common/redis/docker-compose.yaml -# - ./common/db/docker-compose.yaml + - ./common/fusionauth/docker-compose.yaml + - ./common/uptime/docker-compose.yaml + - ./common/clickhouse/docker-compose.yaml + - ./common/redis/docker-compose.yaml + - ./common/db/docker-compose.yaml # - ./common/registry/docker-compose.yaml + - ./bhasai/docker-compose.SDC.yaml services: From 035aed82094c789d3c71c06ada59ad3c9d1d40c0 Mon Sep 17 00:00:00 2001 From: Karun Agarwal Date: Tue, 6 Aug 2024 12:39:53 +0530 Subject: [PATCH 10/28] fix the name of flowise service --- bhasai/flowise/docker-compose.SDC.yaml | 2 +- bhasai/flowise/docker-compose.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bhasai/flowise/docker-compose.SDC.yaml b/bhasai/flowise/docker-compose.SDC.yaml index 1d2319d..6be40a4 100644 --- a/bhasai/flowise/docker-compose.SDC.yaml +++ b/bhasai/flowise/docker-compose.SDC.yaml @@ -1,5 +1,5 @@ services: - admin: + flowise: build: context: https://${GIT_USERNAME}:${GIT_PASSWORD}@github.com/${org}/flowise.git#${FLOWISE_GITHUB_BRANCH:-${DEFAULT_GITHUB_BRANCH:?DEFAULT_GITHUB_BRANCH is not set}} args: diff --git a/bhasai/flowise/docker-compose.yaml b/bhasai/flowise/docker-compose.yaml index d30f01b..0cd73f2 100644 --- a/bhasai/flowise/docker-compose.yaml +++ b/bhasai/flowise/docker-compose.yaml @@ -1,5 +1,5 @@ services: - admin: + flowise: build: context: https://${GIT_USERNAME}:${GIT_PASSWORD}@github.com/${org}/flowise.git#${FLOWISE_GITHUB_BRANCH:-${DEFAULT_GITHUB_BRANCH:?DEFAULT_GITHUB_BRANCH is not set}} args: From b2728aa50fb5c26643f1c21c1028455a1ed0cc07 Mon Sep 17 00:00:00 2001 From: Karun Agarwal Date: Tue, 6 Aug 2024 13:44:43 +0530 Subject: [PATCH 11/28] map bhasai as volume in caddy --- docker-compose.SDC.yaml | 3 ++- docker-compose.yaml | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docker-compose.SDC.yaml b/docker-compose.SDC.yaml index c27353b..2dad0ca 100644 --- a/docker-compose.SDC.yaml +++ b/docker-compose.SDC.yaml @@ -16,7 +16,8 @@ services: extends: file: common/caddy/docker-compose.SDC.yaml service: caddy -# volumes: + volumes: + - ./bhasai:/etc/caddy/bhasai # - ./certs:/certs diff --git a/docker-compose.yaml b/docker-compose.yaml index c86d82c..4a81bd7 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -16,7 +16,8 @@ services: extends: file: common/caddy/docker-compose.yaml service: caddy -# volumes: + volumes: + - ./bhasai:/etc/caddy/bhasai # - project-specific-volume-mappings:/etc/caddy/project-name From e68db1180c1a6b73131441f16b0b14ebff99f189 Mon Sep 17 00:00:00 2001 From: Karun Agarwal <113603846+singhalkarun@users.noreply.github.com> Date: Tue, 6 Aug 2024 16:02:06 +0530 Subject: [PATCH 12/28] script to auto generate root sample.env file on PR (#31) Co-authored-by: GitHub Action --- .github/workflows/pre-merge.yaml | 27 ++++++ sample.env | 107 ++++++++++++++++++++++++ scripts/generate-combined-sample-env.sh | 39 +++++++++ scripts/generate-sample-env.sh | 46 ++++++++++ 4 files changed, 219 insertions(+) create mode 100644 .github/workflows/pre-merge.yaml create mode 100644 sample.env create mode 100755 scripts/generate-combined-sample-env.sh create mode 100755 scripts/generate-sample-env.sh diff --git a/.github/workflows/pre-merge.yaml b/.github/workflows/pre-merge.yaml new file mode 100644 index 0000000..32cd7c4 --- /dev/null +++ b/.github/workflows/pre-merge.yaml @@ -0,0 +1,27 @@ +name: Pre-Merge Update + +on: + pull_request: + types: [opened, synchronize, reopened] + +jobs: + update-code: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + ref: ${{ github.head_ref }} + + - name: Run custom update script + run: | + ./scripts/generate-combined-sample-env.sh + + - name: Commit changes + run: | + git config --global user.name "GitHub Action" + git config --global user.email "action@github.com" + git add . + git commit -m "Update code before merge" || echo "No changes to commit" + git push origin ${{ github.head_ref }} diff --git a/sample.env b/sample.env new file mode 100644 index 0000000..393564f --- /dev/null +++ b/sample.env @@ -0,0 +1,107 @@ +ADMIN_API_KEY +AI_TOOLS_EMBEDDING_BATCH_SIZE +ALLOW_DEFAULT_OTP +APP_JWKS_URI +ASR_SERVICE_PRIORITY +AZURE_SUBSCRIPTION_KEY +AZURE_TRANSLATE_KEY +AZURE_URL +BHASHINI_API_KEY +BHASHINI_ULCA_API_KEY +BHASHINI_USERID +CELERY_MAX_RETRIES +CHAT_WITH_DOCS_ORG_ID +CHAT_WITH_DOCS_ORG_OWNER_EMAIL +CHAT_WITH_DOCS_ORG_OWNER_PASSWORD +CHAT_WITH_DOCS_PASSWORD_KEY +CLICKHOUSE_DB +CLICKHOUSE_PASSWORD +CLICKHOUSE_USER +DEFAULT_OTP +DEFAULT_OTP_USERS +DEFAULT_RETRY_COUNT +DEFAULT_TIMEOUT +DEFAULT_USER_PASSWORD +DEMO_ENV_WITH_DEFAULT +DEMO_ENV_WITH_ERROR +DEMO_SERVICE_REPLICAS +DEPLOYER_APP_ENV +DEPLOYER_PWA_REPO +DEPLOYER_VERCEL_TOKEN +DISCORD_WEBHOOK_URL +DJANGO_SECRET_KEY +DOCKER_REGISTRY_URL +DOMAIN_NAME +DOMAIN_SCHEME +ENABLE_IMAGE_PARSING +ENABLE_SUMMARIZE +ENABLE_TRANSLATE +ENVIRONMENT_PASSWORD +ENVIRONMENT_USERNAME +ES_JAVA_OPTS +FUSIONAUTH_ADMIN_EMAIL +FUSIONAUTH_ADMIN_PASSWORD +FUSIONAUTH_API_KEY +FUSIONAUTH_APP_ID +FUSIONAUTH_APP_RUNTIME_MODE +FUSIONAUTH_KEY +FUSIONAUTH_POSTGRES_PASSWORD +FUSIONAUTH_POSTGRES_ROOT_PASSWORD +FUSIONAUTH_POSTGRES_ROOT_USERNAME +FUSIONAUTH_POSTGRES_USERNAME +FUSIONAUTH_SALT +GF_SECURITY_ADMIN_PASSWORD +GF_SECURITY_ADMIN_USER +GF_SERVER_SERVE_FROM_SUB_PATH +GITHUB_PAT +GIT_PASSWORD +GIT_USERNAME +GRAFANA_DISCORD_WEBHOOK_URL +GUPSHUP_OTP_TEMPLATE +GUPSHUP_PASSWORD +GUPSHUP_USERNAME +HUGGING_FACE_TOKEN +HUGGING_FACE_USERNAME +LOKI_TABLE_INDEX_PERIOD +MINIO_ACCESS_KEY +MINIO_ROOT_PASSWORD +MINIO_ROOT_USER +MINIO_SECRET_KEY +NEXT_PUBLIC_BOT_ID +NEXT_PUBLIC_BOT_NAME +NEXT_PUBLIC_CLARITY_PROJECT_ID +NEXT_PUBLIC_DEBUG +NEXT_PUBLIC_ENTITY_DATASET_ID +NEXT_PUBLIC_FLOWISE_URL +NEXT_PUBLIC_GOOGLE_KEY +NEXT_PUBLIC_ORG_ID +NEXT_PUBLIC_SHOW_ONBOARDING +NEXT_PUBLIC_USER_SERVICE_APP_ID +NEXT_PUBLIC_WEATHER_API +OPENAI_API_KEY +OTP_EXPIRY +PARALLEL_FACTOR +PDF_PARSER_MAX_RETRIES +PDF_PARSER_SECRET_KEY +POSTGRES_PASSWORD +POSTGRES_USER +REGISTRY_BASE_URL +REGISTRY_URL +SPLIT +TLS_ENTRY +TORCH_DEVICE +TRANSLATION_SERVICES +TRANSPORT_SOCKET_DEBUG +UCI_ADMIN_TOKEN +ULCA_API_KEY +ULCA_CONFIG_URL +ULCA_USER_ID +USER_SERVICE_ADMIN_API_KEY +USER_SERVICE_URL +VAULT_PASSWORD +VAULT_USERNAME +WEATHER_VISUAL_CROSSING_API_KEY +WEATHER_VISUAL_CROSSING_BASE_URL +YOUTUBE_DATA_API_KEY +env +org diff --git a/scripts/generate-combined-sample-env.sh b/scripts/generate-combined-sample-env.sh new file mode 100755 index 0000000..c2d2d85 --- /dev/null +++ b/scripts/generate-combined-sample-env.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +# Base directory to search for Docker Compose YAML files +BASE_DIRECTORY="${1:-..}" + +# Output file +OUTPUT_FILE="sample.env" + +# Clear the output file +> "$OUTPUT_FILE" + +# Function to extract environment variables from a docker-compose file +extract_env_vars() { + echo "Processing file: $1" + # Extract variables and their defaults, then filter out unwanted ones + grep -Eo '\${[^}]+}' "$1" | sed 's/[${}]//g' | \ + grep -Eo '^[^:-]+' | sort -u | \ + grep -Ev '(MEM_LIMIT|CPU_LIMIT|IMAGE_TAG|GITHUB_BRANCH)$' >> "$OUTPUT_FILE" +} + +# Check if the base directory exists +if [[ ! -d "$BASE_DIRECTORY" ]]; then + echo "Directory $BASE_DIRECTORY does not exist." + exit 1 +fi + +# Find all matching files in the base directory and its subdirectories +find "$BASE_DIRECTORY" -type f -name 'docker-compose*yaml' | while read -r file; do + extract_env_vars "$file" +done + +# Check if OUTPUT_FILE has content +if [[ -s "$OUTPUT_FILE" ]]; then + # Remove duplicate entries + sort -u "$OUTPUT_FILE" -o "$OUTPUT_FILE" + echo "Environment variables have been extracted to $OUTPUT_FILE" +else + echo "No environment variables found in the specified files." +fi \ No newline at end of file diff --git a/scripts/generate-sample-env.sh b/scripts/generate-sample-env.sh new file mode 100755 index 0000000..65161f5 --- /dev/null +++ b/scripts/generate-sample-env.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +# Base directory to search for Docker Compose YAML files +BASE_DIRECTORY="${1:-..}" + +# Function to extract environment variables from a docker-compose file +extract_env_vars() { + echo "Processing file: $2" + + local tempfile=$(mktemp) + + + # Extract variables and their defaults, then filter out unwanted ones + grep -Eo '\${[^}]+}' "$2" | sed 's/[${}]//g' | \ + grep -Eo '^[^:-]+' | sort -u | \ + grep -Ev '(MEM_LIMIT|CPU_LIMIT|IMAGE_TAG|GITHUB_BRANCH)$' >> "$tempfile" + + # Check if the tempfile has content and is not empty + if [[ -s "$tempfile" ]]; then + local output_file="$1/sample.env" + # Clear the output file if it exists + > "$output_file" + # Remove duplicate entries + sort -u "$tempfile" -o "$output_file" + echo "Environment variables have been extracted to $output_file" + else + echo "No environment variables found in the specified file." + fi +} + +# Check if the base directory exists +if [[ ! -d "$BASE_DIRECTORY" ]]; then + echo "Directory $BASE_DIRECTORY does not exist." + exit 1 +fi + +# Find all matching files in the base directory and its subdirectories +find "$BASE_DIRECTORY" -type f -name 'docker-compose*yaml' | while read -r file; do + # Get the directory of the current docker-compose file + dir=$(dirname "$file") + # Extract environment variables and create sample.env in the corresponding directory + extract_env_vars "$dir" "$file" +done + +# Inform the user +echo "Processing completed. Check individual folders for sample.env files." \ No newline at end of file From b0d85b085deccab51fafe905762f1d87b59b5618 Mon Sep 17 00:00:00 2001 From: Karun Agarwal <113603846+singhalkarun@users.noreply.github.com> Date: Tue, 6 Aug 2024 16:27:58 +0530 Subject: [PATCH 13/28] removed redundant environment variables (#32) Co-authored-by: GitHub Action --- bhasai/bff/docker-compose.SDC.yaml | 15 +++------------ bhasai/bff/docker-compose.yaml | 13 ++----------- bhasai/bff/sample.env | 10 ---------- sample.env | 10 ---------- 4 files changed, 5 insertions(+), 43 deletions(-) diff --git a/bhasai/bff/docker-compose.SDC.yaml b/bhasai/bff/docker-compose.SDC.yaml index f96c686..e0163ca 100644 --- a/bhasai/bff/docker-compose.SDC.yaml +++ b/bhasai/bff/docker-compose.SDC.yaml @@ -13,7 +13,6 @@ services: BFF_BASE_URL: http://bff:3000 EXCEL_PARSER_URL: http://excel_parser:8000 DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/bff?sslmode=disable&connect_timeout=100" - JWKS_URI: ${APP_JWKS_URI} MINIO_PDF_BUCKET: bff MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} MINIO_ENDPOINT: ${DOMAIN_SCHEME:-https}://cdn-api.${DOMAIN_NAME} @@ -26,7 +25,6 @@ services: REDIS_DB: 8 REDIS_HOST: redis REDIS_PORT: 6379 - USER_SERVICE_BASE_URL: http://user_service:3000 OUTBOUND_BASE_URL: http://inboundjs:3000/outbound TRANSPORT_SOCKET_BASE_URL: http://transport_socket:3000 UCI_APIS_BASE_URL: http://uci_apis:3000 @@ -34,15 +32,8 @@ services: ULCA_CONFIG_URL: ${ULCA_CONFIG_URL} ULCA_USER_ID: ${ULCA_USER_ID} YT_PARSER_URL: http://yt_parser:8000 - CHAT_WITH_DOCS_ORG_OWNER_EMAIL: ${CHAT_WITH_DOCS_ORG_OWNER_EMAIL} - CHAT_WITH_DOCS_ORG_OWNER_PASSWORD: ${CHAT_WITH_DOCS_ORG_OWNER_PASSWORD} - CHAT_WITH_DOCS_PASSWORD_KEY: ${CHAT_WITH_DOCS_PASSWORD_KEY} - CHAT_WITH_DOCS_ORG_ID: ${CHAT_WITH_DOCS_ORG_ID} - REGISTRY_URL: ${REGISTRY_URL} + REGISTRY_URL: https://github.com/BharatSahAIyak/packages/blob/main/packages/transformers/registry.json INBOUNDJS_BASE_URL: http://inboundjs:3000 - WEATHER_VISUAL_CROSSING_BASE_URL: ${WEATHER_VISUAL_CROSSING_BASE_URL} - WEATHER_VISUAL_CROSSING_API_KEY: ${WEATHER_VISUAL_CROSSING_API_KEY} DOCUMENT_SERVICE_BASE_URL: ${DOMAIN_SCHEME:-https}://${DOMAIN_NAME}/document-service - AZURE_URL: ${AZURE_URL} - USER_SERVICE_ADMIN_API_KEY: ${USER_SERVICE_ADMIN_API_KEY} - USER_SERVICE_URL: ${USER_SERVICE_URL} + AZURE_URL: ${DOMAIN_SCHEME:-https}://${DOMAIN_NAME}/ai-tools/text_translation/azure_dict/remote/ + USER_SERVICE_ADMIN_API_KEY: ${USER_SERVICE_ADMIN_API_KEY} \ No newline at end of file diff --git a/bhasai/bff/docker-compose.yaml b/bhasai/bff/docker-compose.yaml index 77a76f8..b3553ee 100644 --- a/bhasai/bff/docker-compose.yaml +++ b/bhasai/bff/docker-compose.yaml @@ -13,7 +13,6 @@ services: BFF_BASE_URL: http://bff:3000 EXCEL_PARSER_URL: http://excel_parser:8000 DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/bff?sslmode=disable&connect_timeout=100" - JWKS_URI: ${APP_JWKS_URI} MINIO_PDF_BUCKET: bff MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} MINIO_ENDPOINT: ${DOMAIN_SCHEME:-https}://cdn-api.${DOMAIN_NAME} @@ -26,7 +25,6 @@ services: REDIS_DB: 8 REDIS_HOST: redis REDIS_PORT: 6379 - USER_SERVICE_BASE_URL: http://user_service:3000 OUTBOUND_BASE_URL: http://inboundjs:3000/outbound TRANSPORT_SOCKET_BASE_URL: http://transport_socket:3000 UCI_APIS_BASE_URL: http://uci_apis:3000 @@ -34,15 +32,8 @@ services: ULCA_CONFIG_URL: ${ULCA_CONFIG_URL} ULCA_USER_ID: ${ULCA_USER_ID} YT_PARSER_URL: http://yt_parser:8000 - CHAT_WITH_DOCS_ORG_OWNER_EMAIL: ${CHAT_WITH_DOCS_ORG_OWNER_EMAIL} - CHAT_WITH_DOCS_ORG_OWNER_PASSWORD: ${CHAT_WITH_DOCS_ORG_OWNER_PASSWORD} - CHAT_WITH_DOCS_PASSWORD_KEY: ${CHAT_WITH_DOCS_PASSWORD_KEY} - CHAT_WITH_DOCS_ORG_ID: ${CHAT_WITH_DOCS_ORG_ID} - REGISTRY_URL: ${REGISTRY_URL} + REGISTRY_URL: https://github.com/BharatSahAIyak/packages/blob/main/packages/transformers/registry.json INBOUNDJS_BASE_URL: http://inboundjs:3000 - WEATHER_VISUAL_CROSSING_BASE_URL: ${WEATHER_VISUAL_CROSSING_BASE_URL} - WEATHER_VISUAL_CROSSING_API_KEY: ${WEATHER_VISUAL_CROSSING_API_KEY} DOCUMENT_SERVICE_BASE_URL: ${DOMAIN_SCHEME:-https}://document-service.${DOMAIN_NAME} - AZURE_URL: ${AZURE_URL} + AZURE_URL: ${DOMAIN_SCHEME:-https}://ai-tools.${DOMAIN_NAME}/text_translation/azure_dict/remote/ USER_SERVICE_ADMIN_API_KEY: ${USER_SERVICE_ADMIN_API_KEY} - USER_SERVICE_URL: ${USER_SERVICE_URL} diff --git a/bhasai/bff/sample.env b/bhasai/bff/sample.env index 021c66d..5c884a0 100644 --- a/bhasai/bff/sample.env +++ b/bhasai/bff/sample.env @@ -1,7 +1,6 @@ #bff POSTGRES_USER= POSTGRES_PASSWORD= -APP_JWKS_URI= MINIO_ACCESS_KEY= MINIO_SECRET_KEY= DOMAIN_NAME= @@ -10,13 +9,4 @@ OPENAI_API_KEY= ULCA_API_KEY= ULCA_CONFIG_URL= ULCA_USER_ID= -CHAT_WITH_DOCS_ORG_OWNER_EMAIL= -CHAT_WITH_DOCS_ORG_OWNER_PASSWORD= -CHAT_WITH_DOCS_PASSWORD_KEY= -CHAT_WITH_DOCS_ORG_ID= -REGISTRY_URL= -WEATHER_VISUAL_CROSSING_BASE_URL= -WEATHER_VISUAL_CROSSING_API_KEY= -AZURE_URL= USER_SERVICE_ADMIN_API_KEY= -USER_SERVICE_URL= diff --git a/sample.env b/sample.env index 393564f..3ebb6f3 100644 --- a/sample.env +++ b/sample.env @@ -1,19 +1,13 @@ ADMIN_API_KEY AI_TOOLS_EMBEDDING_BATCH_SIZE ALLOW_DEFAULT_OTP -APP_JWKS_URI ASR_SERVICE_PRIORITY AZURE_SUBSCRIPTION_KEY AZURE_TRANSLATE_KEY -AZURE_URL BHASHINI_API_KEY BHASHINI_ULCA_API_KEY BHASHINI_USERID CELERY_MAX_RETRIES -CHAT_WITH_DOCS_ORG_ID -CHAT_WITH_DOCS_ORG_OWNER_EMAIL -CHAT_WITH_DOCS_ORG_OWNER_PASSWORD -CHAT_WITH_DOCS_PASSWORD_KEY CLICKHOUSE_DB CLICKHOUSE_PASSWORD CLICKHOUSE_USER @@ -86,7 +80,6 @@ PDF_PARSER_SECRET_KEY POSTGRES_PASSWORD POSTGRES_USER REGISTRY_BASE_URL -REGISTRY_URL SPLIT TLS_ENTRY TORCH_DEVICE @@ -97,11 +90,8 @@ ULCA_API_KEY ULCA_CONFIG_URL ULCA_USER_ID USER_SERVICE_ADMIN_API_KEY -USER_SERVICE_URL VAULT_PASSWORD VAULT_USERNAME -WEATHER_VISUAL_CROSSING_API_KEY -WEATHER_VISUAL_CROSSING_BASE_URL YOUTUBE_DATA_API_KEY env org From 3a85c61267edb0c783f78456ac5405190f455f6a Mon Sep 17 00:00:00 2001 From: Karun Agarwal <113603846+singhalkarun@users.noreply.github.com> Date: Tue, 6 Aug 2024 22:53:45 +0530 Subject: [PATCH 14/28] remove redundant/unnecessary variables from sample.env, update sample.env generation script to automatically put default values in sample.env(#33) Co-authored-by: GitHub Action --- .github/workflows/pre-merge.yaml | 2 +- bhasai/admin/docker-compose.SDC.yaml | 2 +- bhasai/admin/docker-compose.yaml | 2 +- bhasai/admin/sample.env | 3 +-- bhasai/bff/docker-compose.SDC.yaml | 2 +- bhasai/bff/docker-compose.yaml | 2 +- bhasai/bff/sample.env | 2 +- bhasai/yt_parser/docker-compose.SDC.yaml | 8 +++--- bhasai/yt_parser/docker-compose.yaml | 8 +++--- bhasai/yt_parser/sample.env | 8 +++--- sample.env | 22 ++++++---------- scripts/generate-combined-sample-env.sh | 32 +++++++++++++++--------- 12 files changed, 47 insertions(+), 46 deletions(-) diff --git a/.github/workflows/pre-merge.yaml b/.github/workflows/pre-merge.yaml index 32cd7c4..789ee4b 100644 --- a/.github/workflows/pre-merge.yaml +++ b/.github/workflows/pre-merge.yaml @@ -16,7 +16,7 @@ jobs: - name: Run custom update script run: | - ./scripts/generate-combined-sample-env.sh + ./scripts/generate-combined-sample-env.sh ./bhasai ./common - name: Commit changes run: | diff --git a/bhasai/admin/docker-compose.SDC.yaml b/bhasai/admin/docker-compose.SDC.yaml index f6bf3dd..b648051 100644 --- a/bhasai/admin/docker-compose.SDC.yaml +++ b/bhasai/admin/docker-compose.SDC.yaml @@ -11,7 +11,7 @@ services: NEXT_PUBLIC_DOC_SERVICE_URL: ${DOMAIN_SCHEME:-https}://${DOMAIN_NAME}/document-service NEXT_PUBLIC_PWA_DEPLOYER_URL: ${DOMAIN_SCHEME:-https}://${DOMAIN_NAME}/deployer NEXT_PUBLIC_TELEMETRY_API_URL: ${DOMAIN_SCHEME:-https}://${DOMAIN_NAME}/telemetry - NEXT_PUBLIC_FLOWISE_URL: ${NEXT_PUBLIC_FLOWISE_URL} + NEXT_PUBLIC_FLOWISE_URL: ${DOMAIN_SCHEME:-https}://${DOMAIN_NAME}/flowise NEXT_PUBLIC_AUTOTUNE_API_URL: ${DOMAIN_SCHEME:-https}://${DOMAIN_NAME}/autotune SERVE_PATH: "/admin" restart: always diff --git a/bhasai/admin/docker-compose.yaml b/bhasai/admin/docker-compose.yaml index 0e4732e..27bde39 100644 --- a/bhasai/admin/docker-compose.yaml +++ b/bhasai/admin/docker-compose.yaml @@ -11,7 +11,7 @@ services: NEXT_PUBLIC_DOC_SERVICE_URL: ${DOMAIN_SCHEME:-https}://document-service.${DOMAIN_NAME} NEXT_PUBLIC_PWA_DEPLOYER_URL: ${DOMAIN_SCHEME:-https}://deployer.${DOMAIN_NAME} NEXT_PUBLIC_TELEMETRY_API_URL: ${DOMAIN_SCHEME:-https}://telemetry.${DOMAIN_NAME} - NEXT_PUBLIC_FLOWISE_URL: ${NEXT_PUBLIC_FLOWISE_URL} + NEXT_PUBLIC_FLOWISE_URL: ${DOMAIN_SCHEME:-https}://flowise.${DOMAIN_NAME} NEXT_PUBLIC_AUTOTUNE_API_URL: ${DOMAIN_SCHEME:-https}://autotune.${DOMAIN_NAME} SERVE_PATH: "" restart: always diff --git a/bhasai/admin/sample.env b/bhasai/admin/sample.env index 0b9ee98..99e423a 100644 --- a/bhasai/admin/sample.env +++ b/bhasai/admin/sample.env @@ -1,4 +1,3 @@ #admin DOMAIN_SCHEME= -DOMAIN_NAME= -NEXT_PUBLIC_FLOWISE_URL= \ No newline at end of file +DOMAIN_NAME= \ No newline at end of file diff --git a/bhasai/bff/docker-compose.SDC.yaml b/bhasai/bff/docker-compose.SDC.yaml index e0163ca..7a92e34 100644 --- a/bhasai/bff/docker-compose.SDC.yaml +++ b/bhasai/bff/docker-compose.SDC.yaml @@ -36,4 +36,4 @@ services: INBOUNDJS_BASE_URL: http://inboundjs:3000 DOCUMENT_SERVICE_BASE_URL: ${DOMAIN_SCHEME:-https}://${DOMAIN_NAME}/document-service AZURE_URL: ${DOMAIN_SCHEME:-https}://${DOMAIN_NAME}/ai-tools/text_translation/azure_dict/remote/ - USER_SERVICE_ADMIN_API_KEY: ${USER_SERVICE_ADMIN_API_KEY} \ No newline at end of file + USER_SERVICE_ADMIN_API_KEY: ${ADMIN_API_KEY} \ No newline at end of file diff --git a/bhasai/bff/docker-compose.yaml b/bhasai/bff/docker-compose.yaml index b3553ee..7998e43 100644 --- a/bhasai/bff/docker-compose.yaml +++ b/bhasai/bff/docker-compose.yaml @@ -36,4 +36,4 @@ services: INBOUNDJS_BASE_URL: http://inboundjs:3000 DOCUMENT_SERVICE_BASE_URL: ${DOMAIN_SCHEME:-https}://document-service.${DOMAIN_NAME} AZURE_URL: ${DOMAIN_SCHEME:-https}://ai-tools.${DOMAIN_NAME}/text_translation/azure_dict/remote/ - USER_SERVICE_ADMIN_API_KEY: ${USER_SERVICE_ADMIN_API_KEY} + USER_SERVICE_ADMIN_API_KEY: ${ADMIN_API_KEY} diff --git a/bhasai/bff/sample.env b/bhasai/bff/sample.env index 5c884a0..09dd326 100644 --- a/bhasai/bff/sample.env +++ b/bhasai/bff/sample.env @@ -9,4 +9,4 @@ OPENAI_API_KEY= ULCA_API_KEY= ULCA_CONFIG_URL= ULCA_USER_ID= -USER_SERVICE_ADMIN_API_KEY= +ADMIN_API_KEY= diff --git a/bhasai/yt_parser/docker-compose.SDC.yaml b/bhasai/yt_parser/docker-compose.SDC.yaml index 6744591..19a02f1 100644 --- a/bhasai/yt_parser/docker-compose.SDC.yaml +++ b/bhasai/yt_parser/docker-compose.SDC.yaml @@ -14,8 +14,8 @@ services: REDIS_CONN: redis://redis:6379/3 CELERY_BROKER_URL: redis://redis:6379/3 CELERY_RESULT_BACKEND: redis://redis:6379/3 - BHASHINI_USERID: ${BHASHINI_USERID} - BHASHINI_ULCA_API_KEY: ${BHASHINI_ULCA_API_KEY} + BHASHINI_USERID: ${ULCA_USER_ID} + BHASHINI_ULCA_API_KEY: ${ULCA_API_KEY} AZURE_SUBSCRIPTION_KEY: ${AZURE_SUBSCRIPTION_KEY} WEBHOOK_URL: http://bff:3000/document/processed-webhook/youtube DISCORD_WEBHOOK_URL: ${DISCORD_WEBHOOK_URL} @@ -40,8 +40,8 @@ services: MINIO_BUCKET: yt_parser MINIO_SECURE_CONN: "False" REDIS_CONN: redis://redis:6379/3 - BHASHINI_USERID: ${BHASHINI_USERID} - BHASHINI_ULCA_API_KEY: ${BHASHINI_ULCA_API_KEY} + BHASHINI_USERID: ${ULCA_USER_ID} + BHASHINI_ULCA_API_KEY: ${ULCA_API_KEY} AZURE_SUBSCRIPTION_KEY: ${AZURE_SUBSCRIPTION_KEY} DISCORD_WEBHOOK_URL: ${DISCORD_WEBHOOK_URL} ASR_SERVICE_PRIORITY: ${ASR_SERVICE_PRIORITY} diff --git a/bhasai/yt_parser/docker-compose.yaml b/bhasai/yt_parser/docker-compose.yaml index 6744591..19a02f1 100644 --- a/bhasai/yt_parser/docker-compose.yaml +++ b/bhasai/yt_parser/docker-compose.yaml @@ -14,8 +14,8 @@ services: REDIS_CONN: redis://redis:6379/3 CELERY_BROKER_URL: redis://redis:6379/3 CELERY_RESULT_BACKEND: redis://redis:6379/3 - BHASHINI_USERID: ${BHASHINI_USERID} - BHASHINI_ULCA_API_KEY: ${BHASHINI_ULCA_API_KEY} + BHASHINI_USERID: ${ULCA_USER_ID} + BHASHINI_ULCA_API_KEY: ${ULCA_API_KEY} AZURE_SUBSCRIPTION_KEY: ${AZURE_SUBSCRIPTION_KEY} WEBHOOK_URL: http://bff:3000/document/processed-webhook/youtube DISCORD_WEBHOOK_URL: ${DISCORD_WEBHOOK_URL} @@ -40,8 +40,8 @@ services: MINIO_BUCKET: yt_parser MINIO_SECURE_CONN: "False" REDIS_CONN: redis://redis:6379/3 - BHASHINI_USERID: ${BHASHINI_USERID} - BHASHINI_ULCA_API_KEY: ${BHASHINI_ULCA_API_KEY} + BHASHINI_USERID: ${ULCA_USER_ID} + BHASHINI_ULCA_API_KEY: ${ULCA_API_KEY} AZURE_SUBSCRIPTION_KEY: ${AZURE_SUBSCRIPTION_KEY} DISCORD_WEBHOOK_URL: ${DISCORD_WEBHOOK_URL} ASR_SERVICE_PRIORITY: ${ASR_SERVICE_PRIORITY} diff --git a/bhasai/yt_parser/sample.env b/bhasai/yt_parser/sample.env index 3357bcc..02f029f 100644 --- a/bhasai/yt_parser/sample.env +++ b/bhasai/yt_parser/sample.env @@ -1,8 +1,8 @@ #yt_parser MINIO_ACCESS_KEY= MINIO_SECRET_KEY= -BHASHINI_USERID= -BHASHINI_ULCA_API_KEY= +ULCA_API_KEY= +ULCA_USER_ID= AZURE_SUBSCRIPTION_KEY= DISCORD_WEBHOOK_URL= ASR_SERVICE_PRIORITY= @@ -11,8 +11,8 @@ YOUTUBE_DATA_API_KEY= #yt_parser_celery_worker MINIO_ACCESS_KEY= MINIO_SECRET_KEY= -BHASHINI_USERID= -BHASHINI_ULCA_API_KEY= +ULCA_USER_ID= +ULCA_API_KEY= AZURE_SUBSCRIPTION_KEY= DISCORD_WEBHOOK_URL= ASR_SERVICE_PRIORITY= diff --git a/sample.env b/sample.env index 3ebb6f3..86c5682 100644 --- a/sample.env +++ b/sample.env @@ -1,30 +1,30 @@ +ALLOW_DEFAULT_OTP=false +DEFAULT_OTP=1234 +DOCKER_REGISTRY_URL=ghcr.io +DOMAIN_SCHEME=https +FUSIONAUTH_APP_RUNTIME_MODE=development +GF_SERVER_SERVE_FROM_SUB_PATH=false +LOKI_TABLE_INDEX_PERIOD=24h +OTP_EXPIRY=30 ADMIN_API_KEY AI_TOOLS_EMBEDDING_BATCH_SIZE -ALLOW_DEFAULT_OTP ASR_SERVICE_PRIORITY AZURE_SUBSCRIPTION_KEY AZURE_TRANSLATE_KEY BHASHINI_API_KEY -BHASHINI_ULCA_API_KEY -BHASHINI_USERID CELERY_MAX_RETRIES CLICKHOUSE_DB CLICKHOUSE_PASSWORD CLICKHOUSE_USER -DEFAULT_OTP DEFAULT_OTP_USERS DEFAULT_RETRY_COUNT DEFAULT_TIMEOUT DEFAULT_USER_PASSWORD -DEMO_ENV_WITH_DEFAULT -DEMO_ENV_WITH_ERROR -DEMO_SERVICE_REPLICAS DEPLOYER_APP_ENV DEPLOYER_PWA_REPO DEPLOYER_VERCEL_TOKEN DISCORD_WEBHOOK_URL DJANGO_SECRET_KEY -DOCKER_REGISTRY_URL DOMAIN_NAME DOMAIN_SCHEME ENABLE_IMAGE_PARSING @@ -37,7 +37,6 @@ FUSIONAUTH_ADMIN_EMAIL FUSIONAUTH_ADMIN_PASSWORD FUSIONAUTH_API_KEY FUSIONAUTH_APP_ID -FUSIONAUTH_APP_RUNTIME_MODE FUSIONAUTH_KEY FUSIONAUTH_POSTGRES_PASSWORD FUSIONAUTH_POSTGRES_ROOT_PASSWORD @@ -46,7 +45,6 @@ FUSIONAUTH_POSTGRES_USERNAME FUSIONAUTH_SALT GF_SECURITY_ADMIN_PASSWORD GF_SECURITY_ADMIN_USER -GF_SERVER_SERVE_FROM_SUB_PATH GITHUB_PAT GIT_PASSWORD GIT_USERNAME @@ -56,7 +54,6 @@ GUPSHUP_PASSWORD GUPSHUP_USERNAME HUGGING_FACE_TOKEN HUGGING_FACE_USERNAME -LOKI_TABLE_INDEX_PERIOD MINIO_ACCESS_KEY MINIO_ROOT_PASSWORD MINIO_ROOT_USER @@ -66,14 +63,12 @@ NEXT_PUBLIC_BOT_NAME NEXT_PUBLIC_CLARITY_PROJECT_ID NEXT_PUBLIC_DEBUG NEXT_PUBLIC_ENTITY_DATASET_ID -NEXT_PUBLIC_FLOWISE_URL NEXT_PUBLIC_GOOGLE_KEY NEXT_PUBLIC_ORG_ID NEXT_PUBLIC_SHOW_ONBOARDING NEXT_PUBLIC_USER_SERVICE_APP_ID NEXT_PUBLIC_WEATHER_API OPENAI_API_KEY -OTP_EXPIRY PARALLEL_FACTOR PDF_PARSER_MAX_RETRIES PDF_PARSER_SECRET_KEY @@ -89,7 +84,6 @@ UCI_ADMIN_TOKEN ULCA_API_KEY ULCA_CONFIG_URL ULCA_USER_ID -USER_SERVICE_ADMIN_API_KEY VAULT_PASSWORD VAULT_USERNAME YOUTUBE_DATA_API_KEY diff --git a/scripts/generate-combined-sample-env.sh b/scripts/generate-combined-sample-env.sh index c2d2d85..10b29d9 100755 --- a/scripts/generate-combined-sample-env.sh +++ b/scripts/generate-combined-sample-env.sh @@ -1,7 +1,7 @@ #!/bin/bash # Base directory to search for Docker Compose YAML files -BASE_DIRECTORY="${1:-..}" +BASE_DIRECTORIES=("${@}") # Output file OUTPUT_FILE="sample.env" @@ -12,27 +12,35 @@ OUTPUT_FILE="sample.env" # Function to extract environment variables from a docker-compose file extract_env_vars() { echo "Processing file: $1" - # Extract variables and their defaults, then filter out unwanted ones - grep -Eo '\${[^}]+}' "$1" | sed 's/[${}]//g' | \ + # Extract variables and their defaults, then filter out unwanted ones + grep -Eo '\${[^}]+}' "$1" | sed -e 's/[${}]//g' -e 's/:-\([^$]*\)/=\1/' | \ grep -Eo '^[^:-]+' | sort -u | \ grep -Ev '(MEM_LIMIT|CPU_LIMIT|IMAGE_TAG|GITHUB_BRANCH)$' >> "$OUTPUT_FILE" } -# Check if the base directory exists -if [[ ! -d "$BASE_DIRECTORY" ]]; then - echo "Directory $BASE_DIRECTORY does not exist." - exit 1 -fi -# Find all matching files in the base directory and its subdirectories -find "$BASE_DIRECTORY" -type f -name 'docker-compose*yaml' | while read -r file; do - extract_env_vars "$file" +# Iterate over each base directory +for BASE_DIRECTORY in "${BASE_DIRECTORIES[@]}"; do + # Check if the base directory exists + if [[ ! -d "$BASE_DIRECTORY" ]]; then + echo "Directory $BASE_DIRECTORY does not exist." + continue + fi + + # Find all matching files in the base directory and its subdirectories + find "$BASE_DIRECTORY" -type f -name 'docker-compose*yaml' | while read -r file; do + extract_env_vars "$file" + done done # Check if OUTPUT_FILE has content if [[ -s "$OUTPUT_FILE" ]]; then - # Remove duplicate entries + # Remove duplicate entries and sort sort -u "$OUTPUT_FILE" -o "$OUTPUT_FILE" + + # Sort lines with '=' to the top and lines without '=' to the bottom +(grep '=' "$OUTPUT_FILE"; grep -v '=' "$OUTPUT_FILE") > temp_output && mv temp_output "$OUTPUT_FILE" + echo "Environment variables have been extracted to $OUTPUT_FILE" else echo "No environment variables found in the specified files." From d2b9c7de0511dcd18cc22abf75b3fa339e9d6d9c Mon Sep 17 00:00:00 2001 From: Karun Agarwal <113603846+singhalkarun@users.noreply.github.com> Date: Wed, 7 Aug 2024 12:25:19 +0530 Subject: [PATCH 15/28] remove variables with MEM_LIMIT CPU_LIMIT IMAGE_TAG and GITHUB_BRANCH aggressively and set defaults wherver possible (#35) Co-authored-by: GitHub Action --- bhasai/ai_tools_proxy/docker-compose.SDC.yaml | 6 +-- bhasai/ai_tools_proxy/docker-compose.yaml | 6 +-- bhasai/app/docker-compose.SDC.yaml | 4 +- bhasai/app/docker-compose.yaml | 4 +- bhasai/autotune/docker-compose.SDC.yaml | 4 +- bhasai/autotune/docker-compose.yaml | 4 +- .../dataset_service/docker-compose.SDC.yaml | 2 +- bhasai/dataset_service/docker-compose.yaml | 2 +- .../document_service/docker-compose.SDC.yaml | 2 +- bhasai/document_service/docker-compose.yaml | 2 +- bhasai/inboundjs/docker-compose.SDC.yaml | 2 +- bhasai/inboundjs/docker-compose.yaml | 2 +- bhasai/nodehub/docker-compose.SDC.yaml | 4 +- bhasai/nodehub/docker-compose.yaml | 4 +- bhasai/nodehub/sample.env | 1 - bhasai/orchestrator/docker-compose.SDC.yaml | 2 +- bhasai/orchestrator/docker-compose.yaml | 2 +- bhasai/pdf_parser/docker-compose.SDC.yaml | 24 +++++------- bhasai/pdf_parser/docker-compose.yaml | 24 +++++------- bhasai/telemetry/docker-compose.SDC.yaml | 6 +-- bhasai/telemetry/docker-compose.yaml | 6 +-- bhasai/user_service/docker-compose.SDC.yaml | 2 +- bhasai/user_service/docker-compose.yaml | 2 +- sample.env | 38 ++++++++++--------- scripts/generate-combined-sample-env.sh | 4 +- 25 files changed, 76 insertions(+), 83 deletions(-) diff --git a/bhasai/ai_tools_proxy/docker-compose.SDC.yaml b/bhasai/ai_tools_proxy/docker-compose.SDC.yaml index a6cefc7..08e95b3 100644 --- a/bhasai/ai_tools_proxy/docker-compose.SDC.yaml +++ b/bhasai/ai_tools_proxy/docker-compose.SDC.yaml @@ -6,8 +6,8 @@ cpus: ${AI_TOOLS_PROXY_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} environment: DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/ai_tools_proxy" - DEFAULT_RETRY_COUNT: ${DEFAULT_RETRY_COUNT} - DEFAULT_TIMEOUT: ${DEFAULT_TIMEOUT} + DEFAULT_RETRY_COUNT: ${DEFAULT_RETRY_COUNT:-3} + DEFAULT_TIMEOUT: ${DEFAULT_TIMEOUT:-30000} ULCA_API_KEY: ${ULCA_API_KEY} ULCA_CONFIG_URL: ${ULCA_CONFIG_URL} ULCA_USER_ID: ${ULCA_USER_ID} @@ -17,6 +17,6 @@ MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} MINIO_BUCKET: ai-tools-proxy - ENVIRONMENT: ${env} + ENVIRONMENT: ${env:-dev} TELEMETRY_URL: http://telemetry:3000 OPENAI_API_KEY: ${OPENAI_API_KEY} diff --git a/bhasai/ai_tools_proxy/docker-compose.yaml b/bhasai/ai_tools_proxy/docker-compose.yaml index a6cefc7..08e95b3 100644 --- a/bhasai/ai_tools_proxy/docker-compose.yaml +++ b/bhasai/ai_tools_proxy/docker-compose.yaml @@ -6,8 +6,8 @@ cpus: ${AI_TOOLS_PROXY_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} environment: DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/ai_tools_proxy" - DEFAULT_RETRY_COUNT: ${DEFAULT_RETRY_COUNT} - DEFAULT_TIMEOUT: ${DEFAULT_TIMEOUT} + DEFAULT_RETRY_COUNT: ${DEFAULT_RETRY_COUNT:-3} + DEFAULT_TIMEOUT: ${DEFAULT_TIMEOUT:-30000} ULCA_API_KEY: ${ULCA_API_KEY} ULCA_CONFIG_URL: ${ULCA_CONFIG_URL} ULCA_USER_ID: ${ULCA_USER_ID} @@ -17,6 +17,6 @@ MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} MINIO_BUCKET: ai-tools-proxy - ENVIRONMENT: ${env} + ENVIRONMENT: ${env:-dev} TELEMETRY_URL: http://telemetry:3000 OPENAI_API_KEY: ${OPENAI_API_KEY} diff --git a/bhasai/app/docker-compose.SDC.yaml b/bhasai/app/docker-compose.SDC.yaml index c3fd649..fd2f6ab 100644 --- a/bhasai/app/docker-compose.SDC.yaml +++ b/bhasai/app/docker-compose.SDC.yaml @@ -14,7 +14,7 @@ services: NEXT_PUBLIC_JWKS_URI: 'http://fusionauth:9011/.well-known/jwks.json' NEXT_PUBLIC_USER_SERVICE_URL: ${DOMAIN_SCHEME:-https}://${DOMAIN_NAME}/user-service NEXT_PUBLIC_ORG_ID: ${NEXT_PUBLIC_ORG_ID} - NEXT_PUBLIC_DEBUG: ${NEXT_PUBLIC_DEBUG} + NEXT_PUBLIC_DEBUG: ${NEXT_PUBLIC_DEBUG:-false} NEXT_PUBLIC_SOCKET_PATH: '/transport-socket/socket.io' NEXT_PUBLIC_GOOGLE_KEY: ${NEXT_PUBLIC_GOOGLE_KEY} FUSIONAUTH_URL: http://fusionauth:9011/ @@ -22,6 +22,6 @@ services: NEXT_PUBLIC_DATASET_URL: ${DOMAIN_SCHEME:-https}://${DOMAIN_NAME}/dataset-service NEXT_PUBLIC_ENTITY_DATASET_ID: ${NEXT_PUBLIC_ENTITY_DATASET_ID} NEXT_PUBLIC_CLARITY_PROJECT_ID: ${NEXT_PUBLIC_CLARITY_PROJECT_ID} - NEXT_PUBLIC_SHOW_ONBOARDING: ${NEXT_PUBLIC_SHOW_ONBOARDING} + NEXT_PUBLIC_SHOW_ONBOARDING: ${NEXT_PUBLIC_SHOW_ONBOARDING:-false} NEXT_PUBLIC_WEATHER_API: ${NEXT_PUBLIC_WEATHER_API} \ No newline at end of file diff --git a/bhasai/app/docker-compose.yaml b/bhasai/app/docker-compose.yaml index ae03abd..e584a81 100644 --- a/bhasai/app/docker-compose.yaml +++ b/bhasai/app/docker-compose.yaml @@ -14,7 +14,7 @@ services: NEXT_PUBLIC_JWKS_URI: 'http://fusionauth:9011/.well-known/jwks.json' NEXT_PUBLIC_USER_SERVICE_URL: ${DOMAIN_SCHEME:-https}://user-service.${DOMAIN_NAME} NEXT_PUBLIC_ORG_ID: ${NEXT_PUBLIC_ORG_ID} - NEXT_PUBLIC_DEBUG: ${NEXT_PUBLIC_DEBUG} + NEXT_PUBLIC_DEBUG: ${NEXT_PUBLIC_DEBUG:-false} NEXT_PUBLIC_SOCKET_PATH: '' NEXT_PUBLIC_GOOGLE_KEY: ${NEXT_PUBLIC_GOOGLE_KEY} FUSIONAUTH_URL: http://fusionauth:9011/ @@ -22,6 +22,6 @@ services: NEXT_PUBLIC_DATASET_URL: ${DOMAIN_SCHEME:-https}://dataset-service.${DOMAIN_NAME} NEXT_PUBLIC_ENTITY_DATASET_ID: ${NEXT_PUBLIC_ENTITY_DATASET_ID} NEXT_PUBLIC_CLARITY_PROJECT_ID: ${NEXT_PUBLIC_CLARITY_PROJECT_ID} - NEXT_PUBLIC_SHOW_ONBOARDING: ${NEXT_PUBLIC_SHOW_ONBOARDING} + NEXT_PUBLIC_SHOW_ONBOARDING: ${NEXT_PUBLIC_SHOW_ONBOARDING:-false} NEXT_PUBLIC_WEATHER_API: ${NEXT_PUBLIC_WEATHER_API} \ No newline at end of file diff --git a/bhasai/autotune/docker-compose.SDC.yaml b/bhasai/autotune/docker-compose.SDC.yaml index 00f470f..960879b 100644 --- a/bhasai/autotune/docker-compose.SDC.yaml +++ b/bhasai/autotune/docker-compose.SDC.yaml @@ -14,7 +14,7 @@ MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} MINIO_BUCKET: autotune MINIO_SECURE_CONN: "True" - CELERY_MAX_RETRIES: ${CELERY_MAX_RETRIES} + CELERY_MAX_RETRIES: ${CELERY_MAX_RETRIES:-3} DJANGO_SECRET_KEY: ${DJANGO_SECRET_KEY} HUGGING_FACE_USERNAME: ${HUGGING_FACE_USERNAME} HUGGING_FACE_TOKEN: ${HUGGING_FACE_TOKEN} @@ -42,7 +42,7 @@ MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} MINIO_BUCKET: autotune MINIO_SECURE_CONN: "True" - CELERY_MAX_RETRIES: ${CELERY_MAX_RETRIES} + CELERY_MAX_RETRIES: ${CELERY_MAX_RETRIES:-3} DJANGO_SECRET_KEY: ${DJANGO_SECRET_KEY} HUGGING_FACE_USERNAME: ${HUGGING_FACE_USERNAME} HUGGING_FACE_TOKEN: ${HUGGING_FACE_TOKEN} diff --git a/bhasai/autotune/docker-compose.yaml b/bhasai/autotune/docker-compose.yaml index 00f470f..960879b 100644 --- a/bhasai/autotune/docker-compose.yaml +++ b/bhasai/autotune/docker-compose.yaml @@ -14,7 +14,7 @@ MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} MINIO_BUCKET: autotune MINIO_SECURE_CONN: "True" - CELERY_MAX_RETRIES: ${CELERY_MAX_RETRIES} + CELERY_MAX_RETRIES: ${CELERY_MAX_RETRIES:-3} DJANGO_SECRET_KEY: ${DJANGO_SECRET_KEY} HUGGING_FACE_USERNAME: ${HUGGING_FACE_USERNAME} HUGGING_FACE_TOKEN: ${HUGGING_FACE_TOKEN} @@ -42,7 +42,7 @@ MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} MINIO_BUCKET: autotune MINIO_SECURE_CONN: "True" - CELERY_MAX_RETRIES: ${CELERY_MAX_RETRIES} + CELERY_MAX_RETRIES: ${CELERY_MAX_RETRIES:-3} DJANGO_SECRET_KEY: ${DJANGO_SECRET_KEY} HUGGING_FACE_USERNAME: ${HUGGING_FACE_USERNAME} HUGGING_FACE_TOKEN: ${HUGGING_FACE_TOKEN} diff --git a/bhasai/dataset_service/docker-compose.SDC.yaml b/bhasai/dataset_service/docker-compose.SDC.yaml index 2416049..c075712 100644 --- a/bhasai/dataset_service/docker-compose.SDC.yaml +++ b/bhasai/dataset_service/docker-compose.SDC.yaml @@ -16,4 +16,4 @@ services: REDIS_PORT: 6379 REDIS_DB: 9 AI_TOOLS_EMBEDDING_BASE_URL: ${DOMAIN_SCHEME:-https}://ai-tools.${DOMAIN_NAME} - AI_TOOLS_BATCH_SIZE: ${AI_TOOLS_EMBEDDING_BATCH_SIZE} + AI_TOOLS_BATCH_SIZE: ${AI_TOOLS_EMBEDDING_BATCH_SIZE:-100} diff --git a/bhasai/dataset_service/docker-compose.yaml b/bhasai/dataset_service/docker-compose.yaml index 2416049..c075712 100644 --- a/bhasai/dataset_service/docker-compose.yaml +++ b/bhasai/dataset_service/docker-compose.yaml @@ -16,4 +16,4 @@ services: REDIS_PORT: 6379 REDIS_DB: 9 AI_TOOLS_EMBEDDING_BASE_URL: ${DOMAIN_SCHEME:-https}://ai-tools.${DOMAIN_NAME} - AI_TOOLS_BATCH_SIZE: ${AI_TOOLS_EMBEDDING_BATCH_SIZE} + AI_TOOLS_BATCH_SIZE: ${AI_TOOLS_EMBEDDING_BATCH_SIZE:-100} diff --git a/bhasai/document_service/docker-compose.SDC.yaml b/bhasai/document_service/docker-compose.SDC.yaml index 53700ba..f5554cc 100644 --- a/bhasai/document_service/docker-compose.SDC.yaml +++ b/bhasai/document_service/docker-compose.SDC.yaml @@ -19,4 +19,4 @@ services: REDIS_PORT: 6379 REDIS_DB: 4 AI_TOOLS_EMBEDDING_BASE_URL: ${DOMAIN_SCHEME:-https}://ai-tools.${DOMAIN_NAME} - AI_TOOLS_BATCH_SIZE: ${AI_TOOLS_EMBEDDING_BATCH_SIZE} + AI_TOOLS_BATCH_SIZE: ${AI_TOOLS_EMBEDDING_BATCH_SIZE:-100} diff --git a/bhasai/document_service/docker-compose.yaml b/bhasai/document_service/docker-compose.yaml index 53700ba..f5554cc 100644 --- a/bhasai/document_service/docker-compose.yaml +++ b/bhasai/document_service/docker-compose.yaml @@ -19,4 +19,4 @@ services: REDIS_PORT: 6379 REDIS_DB: 4 AI_TOOLS_EMBEDDING_BASE_URL: ${DOMAIN_SCHEME:-https}://ai-tools.${DOMAIN_NAME} - AI_TOOLS_BATCH_SIZE: ${AI_TOOLS_EMBEDDING_BATCH_SIZE} + AI_TOOLS_BATCH_SIZE: ${AI_TOOLS_EMBEDDING_BATCH_SIZE:-100} diff --git a/bhasai/inboundjs/docker-compose.SDC.yaml b/bhasai/inboundjs/docker-compose.SDC.yaml index 56ab024..5f661d1 100644 --- a/bhasai/inboundjs/docker-compose.SDC.yaml +++ b/bhasai/inboundjs/docker-compose.SDC.yaml @@ -10,4 +10,4 @@ services: UCI_API_BASE_URL: http://uci-apis:3000 UCI_API_TOKEN: ${UCI_ADMIN_TOKEN} TELEMETRY_SERVICE_HOST: http://telemetry:3000 - ENVIRONMENT: ${env} \ No newline at end of file + ENVIRONMENT: ${env:-dev} \ No newline at end of file diff --git a/bhasai/inboundjs/docker-compose.yaml b/bhasai/inboundjs/docker-compose.yaml index 56ab024..5f661d1 100644 --- a/bhasai/inboundjs/docker-compose.yaml +++ b/bhasai/inboundjs/docker-compose.yaml @@ -10,4 +10,4 @@ services: UCI_API_BASE_URL: http://uci-apis:3000 UCI_API_TOKEN: ${UCI_ADMIN_TOKEN} TELEMETRY_SERVICE_HOST: http://telemetry:3000 - ENVIRONMENT: ${env} \ No newline at end of file + ENVIRONMENT: ${env:-dev} \ No newline at end of file diff --git a/bhasai/nodehub/docker-compose.SDC.yaml b/bhasai/nodehub/docker-compose.SDC.yaml index 413697a..e619d1d 100644 --- a/bhasai/nodehub/docker-compose.SDC.yaml +++ b/bhasai/nodehub/docker-compose.SDC.yaml @@ -7,5 +7,5 @@ services: environment: DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/nodehub" AUTH_SERVICE_URL: "http://auth_service:3000" - REGISTRY_BASE_URL: ${REGISTRY_BASE_URL} - ENVIRONMENT: ${env} \ No newline at end of file + REGISTRY_BASE_URL: https://raw.githubusercontent.com/BharatSahAIyak/packages/main/packages/transformers/registry.json + ENVIRONMENT: ${env:-dev} \ No newline at end of file diff --git a/bhasai/nodehub/docker-compose.yaml b/bhasai/nodehub/docker-compose.yaml index 413697a..e619d1d 100644 --- a/bhasai/nodehub/docker-compose.yaml +++ b/bhasai/nodehub/docker-compose.yaml @@ -7,5 +7,5 @@ services: environment: DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/nodehub" AUTH_SERVICE_URL: "http://auth_service:3000" - REGISTRY_BASE_URL: ${REGISTRY_BASE_URL} - ENVIRONMENT: ${env} \ No newline at end of file + REGISTRY_BASE_URL: https://raw.githubusercontent.com/BharatSahAIyak/packages/main/packages/transformers/registry.json + ENVIRONMENT: ${env:-dev} \ No newline at end of file diff --git a/bhasai/nodehub/sample.env b/bhasai/nodehub/sample.env index dec1330..973cdcb 100644 --- a/bhasai/nodehub/sample.env +++ b/bhasai/nodehub/sample.env @@ -1,3 +1,2 @@ #nodehub -REGISTRY_BASE_URL= env= \ No newline at end of file diff --git a/bhasai/orchestrator/docker-compose.SDC.yaml b/bhasai/orchestrator/docker-compose.SDC.yaml index cd02821..820a43b 100644 --- a/bhasai/orchestrator/docker-compose.SDC.yaml +++ b/bhasai/orchestrator/docker-compose.SDC.yaml @@ -6,4 +6,4 @@ services: restart: always environment: OUTBOUND_SERVICE_BASE_URL: "http://inboundjs:3000/outbound" - ENVIRONMENT: ${env} + ENVIRONMENT: ${env:-dev} diff --git a/bhasai/orchestrator/docker-compose.yaml b/bhasai/orchestrator/docker-compose.yaml index cd02821..820a43b 100644 --- a/bhasai/orchestrator/docker-compose.yaml +++ b/bhasai/orchestrator/docker-compose.yaml @@ -6,4 +6,4 @@ services: restart: always environment: OUTBOUND_SERVICE_BASE_URL: "http://inboundjs:3000/outbound" - ENVIRONMENT: ${env} + ENVIRONMENT: ${env:-dev} diff --git a/bhasai/pdf_parser/docker-compose.SDC.yaml b/bhasai/pdf_parser/docker-compose.SDC.yaml index f82e4ba..6d8d664 100644 --- a/bhasai/pdf_parser/docker-compose.SDC.yaml +++ b/bhasai/pdf_parser/docker-compose.SDC.yaml @@ -15,8 +15,6 @@ services: environment: OPENAI_API_KEY: ${OPENAI_API_KEY} BHASHINI_API_KEY: ${BHASHINI_API_KEY} - WEBHOOK_URL: http://bff:3000/document/processed-webhook - FASTAPI_HOST: http://pdf_parser:8000 TESSDATA_PREFIX: "/usr/share/tesseract-ocr/5/tessdata" MINIO_BASE_URL: ${DOMAIN_SCHEME:-https}://cdn-api.${DOMAIN_NAME} MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} @@ -26,17 +24,17 @@ services: ENABLE_TRANSLATE: ${ENABLE_TRANSLATE} ENABLE_SUMMARIZE: ${ENABLE_SUMMARIZE} ENABLE_IMAGE_PARSING: ${ENABLE_IMAGE_PARSING} - SPLIT: ${SPLIT} - PARALLEL_FACTOR: ${PARALLEL_FACTOR} + SPLIT: ${SPLIT:-5} + PARALLEL_FACTOR: ${PARALLEL_FACTOR:-2} REDIS_CONN: redis://redis:6379/0 - TORCH_DEVICE: ${TORCH_DEVICE} + TORCH_DEVICE: ${TORCH_DEVICE:-cuda} CELERY_BROKER_URL: redis://redis:6379/0 CELERY_RESULT_BACKEND: redis://redis:6379/0 DISCORD_WEBHOOK_URL: ${DISCORD_WEBHOOK_URL} - PDF_PARSER_MAX_RETRIES : ${PDF_PARSER_MAX_RETRIES} + PDF_PARSER_MAX_RETRIES : ${PDF_PARSER_MAX_RETRIES:-3} PDF_PARSER_SECRET_KEY : ${PDF_PARSER_SECRET_KEY} AZURE_TRANSLATE_KEY : ${AZURE_TRANSLATE_KEY} - TRANSLATION_SERVICES : ${TRANSLATION_SERVICES} + TRANSLATION_SERVICES : ${TRANSLATION_SERVICES:-["azure", "bhasini"]} depends_on: - redis @@ -58,8 +56,6 @@ services: CELERY_RESULT_BACKEND: redis://redis:6379/0 OPENAI_API_KEY: ${OPENAI_API_KEY} BHASHINI_API_KEY: ${BHASHINI_API_KEY} - WEBHOOK_URL: http://bff:3000/document/processed-webhook - FASTAPI_HOST: http://pdf_parser:8000 TESSDATA_PREFIX: "/usr/share/tesseract-ocr/5/tessdata" MINIO_BASE_URL: ${DOMAIN_SCHEME:-https}://cdn-api.${DOMAIN_NAME} MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} @@ -69,14 +65,14 @@ services: ENABLE_TRANSLATE: ${ENABLE_TRANSLATE} ENABLE_SUMMARIZE: ${ENABLE_SUMMARIZE} ENABLE_IMAGE_PARSING: ${ENABLE_IMAGE_PARSING} - SPLIT: ${SPLIT} - PARALLEL_FACTOR: ${PARALLEL_FACTOR} + SPLIT: ${SPLIT:-5} + PARALLEL_FACTOR: ${PARALLEL_FACTOR:-2} REDIS_CONN: redis://redis:6379/0 - TORCH_DEVICE: ${TORCH_DEVICE} + TORCH_DEVICE: ${TORCH_DEVICE:-cuda} DISCORD_WEBHOOK_URL: ${DISCORD_WEBHOOK_URL} - PDF_PARSER_MAX_RETRIES : ${PDF_PARSER_MAX_RETRIES} + PDF_PARSER_MAX_RETRIES : ${PDF_PARSER_MAX_RETRIES:-3} PDF_PARSER_SECRET_KEY : ${PDF_PARSER_SECRET_KEY} AZURE_TRANSLATE_KEY : ${AZURE_TRANSLATE_KEY} - TRANSLATION_SERVICES : ${TRANSLATION_SERVICES} + TRANSLATION_SERVICES : ${TRANSLATION_SERVICES:-["azure", "bhasini"]} depends_on: - redis \ No newline at end of file diff --git a/bhasai/pdf_parser/docker-compose.yaml b/bhasai/pdf_parser/docker-compose.yaml index f82e4ba..6d8d664 100644 --- a/bhasai/pdf_parser/docker-compose.yaml +++ b/bhasai/pdf_parser/docker-compose.yaml @@ -15,8 +15,6 @@ services: environment: OPENAI_API_KEY: ${OPENAI_API_KEY} BHASHINI_API_KEY: ${BHASHINI_API_KEY} - WEBHOOK_URL: http://bff:3000/document/processed-webhook - FASTAPI_HOST: http://pdf_parser:8000 TESSDATA_PREFIX: "/usr/share/tesseract-ocr/5/tessdata" MINIO_BASE_URL: ${DOMAIN_SCHEME:-https}://cdn-api.${DOMAIN_NAME} MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} @@ -26,17 +24,17 @@ services: ENABLE_TRANSLATE: ${ENABLE_TRANSLATE} ENABLE_SUMMARIZE: ${ENABLE_SUMMARIZE} ENABLE_IMAGE_PARSING: ${ENABLE_IMAGE_PARSING} - SPLIT: ${SPLIT} - PARALLEL_FACTOR: ${PARALLEL_FACTOR} + SPLIT: ${SPLIT:-5} + PARALLEL_FACTOR: ${PARALLEL_FACTOR:-2} REDIS_CONN: redis://redis:6379/0 - TORCH_DEVICE: ${TORCH_DEVICE} + TORCH_DEVICE: ${TORCH_DEVICE:-cuda} CELERY_BROKER_URL: redis://redis:6379/0 CELERY_RESULT_BACKEND: redis://redis:6379/0 DISCORD_WEBHOOK_URL: ${DISCORD_WEBHOOK_URL} - PDF_PARSER_MAX_RETRIES : ${PDF_PARSER_MAX_RETRIES} + PDF_PARSER_MAX_RETRIES : ${PDF_PARSER_MAX_RETRIES:-3} PDF_PARSER_SECRET_KEY : ${PDF_PARSER_SECRET_KEY} AZURE_TRANSLATE_KEY : ${AZURE_TRANSLATE_KEY} - TRANSLATION_SERVICES : ${TRANSLATION_SERVICES} + TRANSLATION_SERVICES : ${TRANSLATION_SERVICES:-["azure", "bhasini"]} depends_on: - redis @@ -58,8 +56,6 @@ services: CELERY_RESULT_BACKEND: redis://redis:6379/0 OPENAI_API_KEY: ${OPENAI_API_KEY} BHASHINI_API_KEY: ${BHASHINI_API_KEY} - WEBHOOK_URL: http://bff:3000/document/processed-webhook - FASTAPI_HOST: http://pdf_parser:8000 TESSDATA_PREFIX: "/usr/share/tesseract-ocr/5/tessdata" MINIO_BASE_URL: ${DOMAIN_SCHEME:-https}://cdn-api.${DOMAIN_NAME} MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} @@ -69,14 +65,14 @@ services: ENABLE_TRANSLATE: ${ENABLE_TRANSLATE} ENABLE_SUMMARIZE: ${ENABLE_SUMMARIZE} ENABLE_IMAGE_PARSING: ${ENABLE_IMAGE_PARSING} - SPLIT: ${SPLIT} - PARALLEL_FACTOR: ${PARALLEL_FACTOR} + SPLIT: ${SPLIT:-5} + PARALLEL_FACTOR: ${PARALLEL_FACTOR:-2} REDIS_CONN: redis://redis:6379/0 - TORCH_DEVICE: ${TORCH_DEVICE} + TORCH_DEVICE: ${TORCH_DEVICE:-cuda} DISCORD_WEBHOOK_URL: ${DISCORD_WEBHOOK_URL} - PDF_PARSER_MAX_RETRIES : ${PDF_PARSER_MAX_RETRIES} + PDF_PARSER_MAX_RETRIES : ${PDF_PARSER_MAX_RETRIES:-3} PDF_PARSER_SECRET_KEY : ${PDF_PARSER_SECRET_KEY} AZURE_TRANSLATE_KEY : ${AZURE_TRANSLATE_KEY} - TRANSLATION_SERVICES : ${TRANSLATION_SERVICES} + TRANSLATION_SERVICES : ${TRANSLATION_SERVICES:-["azure", "bhasini"]} depends_on: - redis \ No newline at end of file diff --git a/bhasai/telemetry/docker-compose.SDC.yaml b/bhasai/telemetry/docker-compose.SDC.yaml index 0a713a0..3c21638 100644 --- a/bhasai/telemetry/docker-compose.SDC.yaml +++ b/bhasai/telemetry/docker-compose.SDC.yaml @@ -7,7 +7,7 @@ services: environment: DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/telemetry" CLICKHOUSE_HOST: http://clickhouse:8123 - CLICKHOUSE_DB: ${CLICKHOUSE_DB} - CLICKHOUSE_USER: ${CLICKHOUSE_USER} - CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD} + CLICKHOUSE_DB: ${CLICKHOUSE_DB:-default} + CLICKHOUSE_USER: ${CLICKHOUSE_USER:-default} + CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:-} AUTH_SERVICE_URL: http://auth_service:3000 diff --git a/bhasai/telemetry/docker-compose.yaml b/bhasai/telemetry/docker-compose.yaml index 0a713a0..3c21638 100644 --- a/bhasai/telemetry/docker-compose.yaml +++ b/bhasai/telemetry/docker-compose.yaml @@ -7,7 +7,7 @@ services: environment: DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/telemetry" CLICKHOUSE_HOST: http://clickhouse:8123 - CLICKHOUSE_DB: ${CLICKHOUSE_DB} - CLICKHOUSE_USER: ${CLICKHOUSE_USER} - CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD} + CLICKHOUSE_DB: ${CLICKHOUSE_DB:-default} + CLICKHOUSE_USER: ${CLICKHOUSE_USER:-default} + CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:-} AUTH_SERVICE_URL: http://auth_service:3000 diff --git a/bhasai/user_service/docker-compose.SDC.yaml b/bhasai/user_service/docker-compose.SDC.yaml index 94263f8..8d0d4df 100644 --- a/bhasai/user_service/docker-compose.SDC.yaml +++ b/bhasai/user_service/docker-compose.SDC.yaml @@ -8,7 +8,7 @@ services: - ADMIN_API_KEY=${ADMIN_API_KEY} - ALLOW_DEFAULT_OTP=${ALLOW_DEFAULT_OTP:-false} - DEFAULT_OTP=${DEFAULT_OTP:-1234} - - DEFAULT_OTP_USERS=${DEFAULT_OTP_USERS} + - DEFAULT_OTP_USERS=${DEFAULT_OTP_USERS:-[]} - DEFAULT_USER_PASSWORD=${DEFAULT_USER_PASSWORD} - DEFAULT_USER_PASSWORD="http://fusionauth:9011" - FUSIONAUTH_OLD_BASE_URL"="http://fusionauth:9011" diff --git a/bhasai/user_service/docker-compose.yaml b/bhasai/user_service/docker-compose.yaml index 94263f8..8d0d4df 100644 --- a/bhasai/user_service/docker-compose.yaml +++ b/bhasai/user_service/docker-compose.yaml @@ -8,7 +8,7 @@ services: - ADMIN_API_KEY=${ADMIN_API_KEY} - ALLOW_DEFAULT_OTP=${ALLOW_DEFAULT_OTP:-false} - DEFAULT_OTP=${DEFAULT_OTP:-1234} - - DEFAULT_OTP_USERS=${DEFAULT_OTP_USERS} + - DEFAULT_OTP_USERS=${DEFAULT_OTP_USERS:-[]} - DEFAULT_USER_PASSWORD=${DEFAULT_USER_PASSWORD} - DEFAULT_USER_PASSWORD="http://fusionauth:9011" - FUSIONAUTH_OLD_BASE_URL"="http://fusionauth:9011" diff --git a/sample.env b/sample.env index 86c5682..8274043 100644 --- a/sample.env +++ b/sample.env @@ -1,24 +1,36 @@ +AI_TOOLS_EMBEDDING_BATCH_SIZE=100 ALLOW_DEFAULT_OTP=false +CELERY_MAX_RETRIES=3 +CLICKHOUSE_DB=default +CLICKHOUSE_PASSWORD= +CLICKHOUSE_USER=default DEFAULT_OTP=1234 +DEFAULT_OTP_USERS=[] +DEFAULT_RETRY_COUNT=3 +DEFAULT_TIMEOUT=30000 DOCKER_REGISTRY_URL=ghcr.io +DOMAIN_NAME=localdev.me +DOMAIN_SCHEME=http DOMAIN_SCHEME=https +ES_JAVA_OPTS=-Xms2048m -Xmx2048m -XX:InitiatingHeapOccupancyPercent=75 -Djava.awt.headless=true FUSIONAUTH_APP_RUNTIME_MODE=development +GF_SECURITY_ADMIN_USER=admin GF_SERVER_SERVE_FROM_SUB_PATH=false LOKI_TABLE_INDEX_PERIOD=24h +NEXT_PUBLIC_DEBUG=false +NEXT_PUBLIC_SHOW_ONBOARDING=false OTP_EXPIRY=30 +PARALLEL_FACTOR=2 +PDF_PARSER_MAX_RETRIES=3 +SPLIT=5 +TORCH_DEVICE=cuda +TRANSLATION_SERVICES=["azure", "bhasini"] +env=dev ADMIN_API_KEY -AI_TOOLS_EMBEDDING_BATCH_SIZE ASR_SERVICE_PRIORITY AZURE_SUBSCRIPTION_KEY AZURE_TRANSLATE_KEY BHASHINI_API_KEY -CELERY_MAX_RETRIES -CLICKHOUSE_DB -CLICKHOUSE_PASSWORD -CLICKHOUSE_USER -DEFAULT_OTP_USERS -DEFAULT_RETRY_COUNT -DEFAULT_TIMEOUT DEFAULT_USER_PASSWORD DEPLOYER_APP_ENV DEPLOYER_PWA_REPO @@ -32,7 +44,6 @@ ENABLE_SUMMARIZE ENABLE_TRANSLATE ENVIRONMENT_PASSWORD ENVIRONMENT_USERNAME -ES_JAVA_OPTS FUSIONAUTH_ADMIN_EMAIL FUSIONAUTH_ADMIN_PASSWORD FUSIONAUTH_API_KEY @@ -44,7 +55,6 @@ FUSIONAUTH_POSTGRES_ROOT_USERNAME FUSIONAUTH_POSTGRES_USERNAME FUSIONAUTH_SALT GF_SECURITY_ADMIN_PASSWORD -GF_SECURITY_ADMIN_USER GITHUB_PAT GIT_PASSWORD GIT_USERNAME @@ -61,24 +71,16 @@ MINIO_SECRET_KEY NEXT_PUBLIC_BOT_ID NEXT_PUBLIC_BOT_NAME NEXT_PUBLIC_CLARITY_PROJECT_ID -NEXT_PUBLIC_DEBUG NEXT_PUBLIC_ENTITY_DATASET_ID NEXT_PUBLIC_GOOGLE_KEY NEXT_PUBLIC_ORG_ID -NEXT_PUBLIC_SHOW_ONBOARDING NEXT_PUBLIC_USER_SERVICE_APP_ID NEXT_PUBLIC_WEATHER_API OPENAI_API_KEY -PARALLEL_FACTOR -PDF_PARSER_MAX_RETRIES PDF_PARSER_SECRET_KEY POSTGRES_PASSWORD POSTGRES_USER -REGISTRY_BASE_URL -SPLIT TLS_ENTRY -TORCH_DEVICE -TRANSLATION_SERVICES TRANSPORT_SOCKET_DEBUG UCI_ADMIN_TOKEN ULCA_API_KEY diff --git a/scripts/generate-combined-sample-env.sh b/scripts/generate-combined-sample-env.sh index 10b29d9..d9312cb 100755 --- a/scripts/generate-combined-sample-env.sh +++ b/scripts/generate-combined-sample-env.sh @@ -14,8 +14,8 @@ extract_env_vars() { echo "Processing file: $1" # Extract variables and their defaults, then filter out unwanted ones grep -Eo '\${[^}]+}' "$1" | sed -e 's/[${}]//g' -e 's/:-\([^$]*\)/=\1/' | \ - grep -Eo '^[^:-]+' | sort -u | \ - grep -Ev '(MEM_LIMIT|CPU_LIMIT|IMAGE_TAG|GITHUB_BRANCH)$' >> "$OUTPUT_FILE" + sed 's/:?.*$//' | \ + grep -Ev '(MEM_LIMIT|CPU_LIMIT|IMAGE_TAG|GITHUB_BRANCH)' >> "$OUTPUT_FILE" } From 74dda1552d645d78bb5b7e99406372239a418f86 Mon Sep 17 00:00:00 2001 From: Karun Agarwal Date: Wed, 7 Aug 2024 13:13:36 +0530 Subject: [PATCH 16/28] add script to generate env file and add makefile target to run script, update the name of generate-combined-sample-env to generate-sample-env and replace name wherever used, removed generate-sample-env.sh --- .github/workflows/pre-merge.yaml | 2 +- Makefile | 3 + scripts/generate-combined-sample-env.sh | 47 ---------------- scripts/generate-env.sh | 39 +++++++++++++ scripts/generate-sample-env.sh | 73 +++++++++++++------------ 5 files changed, 80 insertions(+), 84 deletions(-) delete mode 100755 scripts/generate-combined-sample-env.sh create mode 100755 scripts/generate-env.sh diff --git a/.github/workflows/pre-merge.yaml b/.github/workflows/pre-merge.yaml index 789ee4b..344dff3 100644 --- a/.github/workflows/pre-merge.yaml +++ b/.github/workflows/pre-merge.yaml @@ -16,7 +16,7 @@ jobs: - name: Run custom update script run: | - ./scripts/generate-combined-sample-env.sh ./bhasai ./common + ./scripts/generate-sample-env.sh ./bhasai ./common - name: Commit changes run: | diff --git a/Makefile b/Makefile index 6f5758b..60cccdb 100644 --- a/Makefile +++ b/Makefile @@ -16,6 +16,9 @@ setup-daemon: setup-webhook: @./scripts/webhook/setup-webhook.sh + +generate-env: + @./scripts/generate-env.sh reload-caddy: @echo "Reloading caddy" diff --git a/scripts/generate-combined-sample-env.sh b/scripts/generate-combined-sample-env.sh deleted file mode 100755 index d9312cb..0000000 --- a/scripts/generate-combined-sample-env.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash - -# Base directory to search for Docker Compose YAML files -BASE_DIRECTORIES=("${@}") - -# Output file -OUTPUT_FILE="sample.env" - -# Clear the output file -> "$OUTPUT_FILE" - -# Function to extract environment variables from a docker-compose file -extract_env_vars() { - echo "Processing file: $1" - # Extract variables and their defaults, then filter out unwanted ones - grep -Eo '\${[^}]+}' "$1" | sed -e 's/[${}]//g' -e 's/:-\([^$]*\)/=\1/' | \ - sed 's/:?.*$//' | \ - grep -Ev '(MEM_LIMIT|CPU_LIMIT|IMAGE_TAG|GITHUB_BRANCH)' >> "$OUTPUT_FILE" -} - - -# Iterate over each base directory -for BASE_DIRECTORY in "${BASE_DIRECTORIES[@]}"; do - # Check if the base directory exists - if [[ ! -d "$BASE_DIRECTORY" ]]; then - echo "Directory $BASE_DIRECTORY does not exist." - continue - fi - - # Find all matching files in the base directory and its subdirectories - find "$BASE_DIRECTORY" -type f -name 'docker-compose*yaml' | while read -r file; do - extract_env_vars "$file" - done -done - -# Check if OUTPUT_FILE has content -if [[ -s "$OUTPUT_FILE" ]]; then - # Remove duplicate entries and sort - sort -u "$OUTPUT_FILE" -o "$OUTPUT_FILE" - - # Sort lines with '=' to the top and lines without '=' to the bottom -(grep '=' "$OUTPUT_FILE"; grep -v '=' "$OUTPUT_FILE") > temp_output && mv temp_output "$OUTPUT_FILE" - - echo "Environment variables have been extracted to $OUTPUT_FILE" -else - echo "No environment variables found in the specified files." -fi \ No newline at end of file diff --git a/scripts/generate-env.sh b/scripts/generate-env.sh new file mode 100755 index 0000000..a855e80 --- /dev/null +++ b/scripts/generate-env.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +# Path to the sample.env file +sample_env_file="sample.env" + +# Path to the generated .env file +generated_env_file=".env" + +# Check if the .env file already exists +if [ -f "$generated_env_file" ]; then + echo ".env file already exists. Generation aborted." + exit 1 +fi + +# Read the sample.env file line by line +while IFS= read -r line; do + # Check if the line contains the pattern "random-generate-" + if [[ $line == *"random-generate-lower"* ]]; then + # Extract the variable name + variable_name=$(echo "$line" | cut -d '=' -f 1) + + # Generate a random value with the specified number of digits + random_value=$(tr -dc a-z0-9 > "$tempfile" - - # Check if the tempfile has content and is not empty - if [[ -s "$tempfile" ]]; then - local output_file="$1/sample.env" - # Clear the output file if it exists - > "$output_file" - # Remove duplicate entries - sort -u "$tempfile" -o "$output_file" - echo "Environment variables have been extracted to $output_file" - else - echo "No environment variables found in the specified file." - fi + echo "Processing file: $1" + # Extract variables and their defaults, then filter out unwanted ones + grep -Eo '\${[^}]+}' "$1" | sed -e 's/[${}]//g' -e 's/:-\([^$]*\)/=\1/' | \ + sed 's/:?.*$//' | \ + grep -Ev '(MEM_LIMIT|CPU_LIMIT|IMAGE_TAG|GITHUB_BRANCH)' >> "$OUTPUT_FILE" } -# Check if the base directory exists -if [[ ! -d "$BASE_DIRECTORY" ]]; then - echo "Directory $BASE_DIRECTORY does not exist." - exit 1 -fi - -# Find all matching files in the base directory and its subdirectories -find "$BASE_DIRECTORY" -type f -name 'docker-compose*yaml' | while read -r file; do - # Get the directory of the current docker-compose file - dir=$(dirname "$file") - # Extract environment variables and create sample.env in the corresponding directory - extract_env_vars "$dir" "$file" + +# Iterate over each base directory +for BASE_DIRECTORY in "${BASE_DIRECTORIES[@]}"; do + # Check if the base directory exists + if [[ ! -d "$BASE_DIRECTORY" ]]; then + echo "Directory $BASE_DIRECTORY does not exist." + continue + fi + + # Find all matching files in the base directory and its subdirectories + find "$BASE_DIRECTORY" -type f -name 'docker-compose*yaml' | while read -r file; do + extract_env_vars "$file" + done done -# Inform the user -echo "Processing completed. Check individual folders for sample.env files." \ No newline at end of file +# Check if OUTPUT_FILE has content +if [[ -s "$OUTPUT_FILE" ]]; then + # Remove duplicate entries and sort + sort -u "$OUTPUT_FILE" -o "$OUTPUT_FILE" + + # Sort lines with '=' to the top and lines without '=' to the bottom +(grep '=' "$OUTPUT_FILE"; grep -v '=' "$OUTPUT_FILE") > temp_output && mv temp_output "$OUTPUT_FILE" + + echo "Environment variables have been extracted to $OUTPUT_FILE" +else + echo "No environment variables found in the specified files." +fi \ No newline at end of file From ecf3467aec9c75b59555336f108df3efeed64871 Mon Sep 17 00:00:00 2001 From: Karun Agarwal <113603846+singhalkarun@users.noreply.github.com> Date: Thu, 8 Aug 2024 16:33:36 +0530 Subject: [PATCH 17/28] fix a typo in Caddyfile.SDC path in uci-apis --- bhasai/uci_apis/Caddyfile.SDC | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bhasai/uci_apis/Caddyfile.SDC b/bhasai/uci_apis/Caddyfile.SDC index 0b50acb..03f273b 100644 --- a/bhasai/uci_apis/Caddyfile.SDC +++ b/bhasai/uci_apis/Caddyfile.SDC @@ -1,3 +1,3 @@ -handle_path /uci_apis* { +handle_path /uci-apis* { reverse_proxy uci_apis:3000 } From ed188d4dab286776d3fa8278ae09a337de0892de Mon Sep 17 00:00:00 2001 From: Karun Agarwal Date: Fri, 9 Aug 2024 12:48:36 +0530 Subject: [PATCH 18/28] add superset from common services to root files --- Caddyfile | 3 ++- docker-compose.SDC.yaml | 2 ++ docker-compose.yaml | 2 ++ 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/Caddyfile b/Caddyfile index 6355cff..955d5c3 100644 --- a/Caddyfile +++ b/Caddyfile @@ -5,4 +5,5 @@ import ./common/fusionauth/Caddyfile # The registry doesn't have a auth thus exposing it publicly means anyone can access the images pushed to this registry # import ./common/registry/Caddyfile import ./common/uptime/Caddyfile -import ./bhasai/Caddyfile \ No newline at end of file +import ./bhasai/Caddyfile +import ./common/superset/Caddyfile \ No newline at end of file diff --git a/docker-compose.SDC.yaml b/docker-compose.SDC.yaml index 2dad0ca..6cfd58e 100644 --- a/docker-compose.SDC.yaml +++ b/docker-compose.SDC.yaml @@ -9,6 +9,8 @@ include: - ./common/db/docker-compose.yaml # - ./common/registry/docker-compose.yaml - ./bhasai/docker-compose.SDC.yaml + - ./common/superset/docker-compose.yaml + services: diff --git a/docker-compose.yaml b/docker-compose.yaml index 4a81bd7..7c30728 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -9,6 +9,8 @@ include: - ./common/db/docker-compose.yaml # - ./common/registry/docker-compose.yaml - ./bhasai/docker-compose.yaml + - ./common/superset/docker-compose.yaml + services: From 5cc25eb6e1d3e54f769e13620b93fafeacb2ebdf Mon Sep 17 00:00:00 2001 From: Karun Agarwal <113603846+singhalkarun@users.noreply.github.com> Date: Mon, 12 Aug 2024 11:40:15 +0530 Subject: [PATCH 19/28] add ai tools (#36) Co-authored-by: GitHub Action --- Makefile | 5 +- bhasai/Caddyfile | 3 +- bhasai/Caddyfile.SDC | 3 +- bhasai/ai_tools/.gitignore | 3 + bhasai/ai_tools/generate.sh | 88 ++++++++++++++ bhasai/docker-compose.SDC.yaml | 3 +- bhasai/docker-compose.yaml | 1 + bhasai/utility/README.md | 1 + bhasai/utility/docker-compose.yaml | 8 ++ sample.env | 184 ++++++++++++++--------------- scripts/generate-sample-env.sh | 6 +- 11 files changed, 207 insertions(+), 98 deletions(-) create mode 100644 bhasai/ai_tools/.gitignore create mode 100755 bhasai/ai_tools/generate.sh create mode 100644 bhasai/utility/README.md create mode 100644 bhasai/utility/docker-compose.yaml diff --git a/Makefile b/Makefile index b406b57..8129efc 100644 --- a/Makefile +++ b/Makefile @@ -5,6 +5,9 @@ SDC := $(if $(filter 1,$(SDC)),-f docker-compose.SDC.yaml,) DOCKER_COMPOSE_COMMAND=docker compose ${SDC} $(REMOVE_ANSI_FLAG) -p bhasai +generate: + @./bhasai/ai_tools/generate.sh + install-docker: @./scripts/install-docker.sh @@ -27,7 +30,7 @@ reload-caddy: @echo "Reloading caddy" $(DOCKER_COMPOSE_COMMAND) exec -w /etc/caddy caddy caddy reload || true -deploy: $(if $(filter 1,$(ENABLE_GIT_PULL)),git-pull,) $(if $(filter 1,$(DISABLE_PULL)),,pull build) reload-caddy +deploy: generate $(if $(filter 1,$(ENABLE_GIT_PULL)),git-pull,) $(if $(filter 1,$(DISABLE_PULL)),,pull build) reload-caddy $(DOCKER_COMPOSE_COMMAND) up -d $(FORCE_RECREATE_FLAG) $(REMOVE_ORPHANS_FLAG) ${services} restart: diff --git a/bhasai/Caddyfile b/bhasai/Caddyfile index b5ff724..0baee12 100644 --- a/bhasai/Caddyfile +++ b/bhasai/Caddyfile @@ -13,4 +13,5 @@ import ./telemetry/Caddyfile import ./transport_socket/Caddyfile import ./uci_apis/Caddyfile import ./user_management_service/Caddyfile -import ./yt_parser/Caddyfile \ No newline at end of file +import ./yt_parser/Caddyfile +import ./ai_tools/Caddyfile \ No newline at end of file diff --git a/bhasai/Caddyfile.SDC b/bhasai/Caddyfile.SDC index 928a37e..b1a1191 100644 --- a/bhasai/Caddyfile.SDC +++ b/bhasai/Caddyfile.SDC @@ -16,4 +16,5 @@ import ./user_management_service/Caddyfile.SDC import ./yt_parser/Caddyfile.SDC import ./flowise/Caddyfile.SDC import ./app/Caddyfile.SDC -import ./user_service/Caddyfile.SDC \ No newline at end of file +import ./user_service/Caddyfile.SDC +import ./ai_tools/Caddyfile.SDC \ No newline at end of file diff --git a/bhasai/ai_tools/.gitignore b/bhasai/ai_tools/.gitignore new file mode 100644 index 0000000..8f63f43 --- /dev/null +++ b/bhasai/ai_tools/.gitignore @@ -0,0 +1,3 @@ +docker-compose.yaml +Caddyfile +Caddyfile.SDC \ No newline at end of file diff --git a/bhasai/ai_tools/generate.sh b/bhasai/ai_tools/generate.sh new file mode 100755 index 0000000..461fac7 --- /dev/null +++ b/bhasai/ai_tools/generate.sh @@ -0,0 +1,88 @@ +#!/bin/bash +set -e + +################################################################################################## +# Description: This script generates Docker Compose and Caddy configuration # +# files based on a AI_SERVICES environment variable # +# # +# Required Environment Variables: # +# - org: GitHub organization # +# - DOCKER_REGISTRY_URL: Docker registry to use for the container images (default: 'ghcr.io') # +################################################################################################## + + +source .env + +required_env=("org" "DOCKER_REGISTRY_URL" "AI_SERVICES") + +# Check if required environment variables are set +for var in "${required_env[@]}"; do + if [ -z "${!var}" ]; then + echo "Error: '$var' environment variable is not set." + exit 1 + fi +done + +directory="bhasai/ai_tools" + +GITHUB_REPOSITORY_URL="${org}/ai-tools" +GITHUB_BRANCH="${AI_TOOLS_GITHUB_BRANCH:-${DEFAULT_GITHUB_BRANCH?DEFAULT_GITHUB_BRANCH is not set}}" + +command -v jq >/dev/null || { echo "Error: jq is required but not found. Please install it."; exit 1; } + +# Get the number of models from the config.json file +count=$(echo "$AI_SERVICES" | jq -r '.models | length') +ComposeFile="${directory}/docker-compose.yaml" +Caddyfile="${directory}/Caddyfile" +CaddyfileSDC="${directory}/Caddyfile.SDC" + +# Generate docker-compose.yaml file +printf "services:\n" > $ComposeFile + +printf '{$DOMAIN_SCHEME}://ai-tools.{$DOMAIN_NAME} {' > $Caddyfile +printf "" > $CaddyfileSDC + +# Loop through each model +for ((i=0; i<$count; i++)); do + # Get model details from config.json + serviceName=$(echo "$AI_SERVICES" | jq -r ".models[$i].serviceName") + apiBasePath=$(echo "$AI_SERVICES" | jq -r ".models[$i].apiBasePath") + containerPort=$(echo "$AI_SERVICES" | jq -r ".models[$i].containerPort") + gpu=$(echo "$AI_SERVICES" | jq ".models[$i].gpu") + + containerImage="${DOCKER_REGISTRY_URL}/${GITHUB_REPOSITORY_URL}/${serviceName}:\${AI_TOOLS_IMAGE_TAG:-\${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}}" + + echo "handle_path /ai-tools/${apiBasePath}* { + reverse_proxy ${serviceName}:${containerPort} +}" >> $CaddyfileSDC + + echo " + handle_path /${apiBasePath}* { + reverse_proxy ${serviceName}:${containerPort} + } + " >> $Caddyfile + + + # Get environment variables for the model + environment=($(echo "$AI_SERVICES" | jq -r ".models[$i].environment | keys[]")) + + # Add service details to docker-compose.yaml + + printf " ${serviceName}:\n image: $containerImage\n restart: always\n" >> $ComposeFile + # Add environment variables to docker-compose.yaml + if [[ ${#environment[@]} -gt 0 ]]; then + printf " environment:\n" >> $ComposeFile + fi + for key in "${environment[@]}"; do + printf " - ${key}=\${${key}}\n" >> $ComposeFile + done + + if [[ "$gpu" == 'true' ]] ;then + + printf " deploy:\n resources:\n reservations:\n devices:\n - driver: nvidia\n count: 1\n capabilities: [gpu]\n" >> $ComposeFile + + fi + printf "\n" >> $ComposeFile + +done +printf "}" >> $Caddyfile diff --git a/bhasai/docker-compose.SDC.yaml b/bhasai/docker-compose.SDC.yaml index af9c63a..763559b 100644 --- a/bhasai/docker-compose.SDC.yaml +++ b/bhasai/docker-compose.SDC.yaml @@ -19,4 +19,5 @@ include: - ./yt_parser/docker-compose.SDC.yaml - ./flowise/docker-compose.SDC.yaml - ./app/docker-compose.SDC.yaml - - ./user_service/docker-compose.SDC.yaml \ No newline at end of file + - ./user_service/docker-compose.SDC.yaml + - ./ai_tools/docker-compose.yaml \ No newline at end of file diff --git a/bhasai/docker-compose.yaml b/bhasai/docker-compose.yaml index ab2cdff..69c44a2 100644 --- a/bhasai/docker-compose.yaml +++ b/bhasai/docker-compose.yaml @@ -17,3 +17,4 @@ include: - ./user_management_service/docker-compose.yaml - ./vault/docker-compose.yaml - ./yt_parser/docker-compose.yaml + - ./ai_tools/docker-compose.yaml diff --git a/bhasai/utility/README.md b/bhasai/utility/README.md new file mode 100644 index 0000000..3b903b3 --- /dev/null +++ b/bhasai/utility/README.md @@ -0,0 +1 @@ +# This folder has fake yamls which allows env processor to generate env which are skipped \ No newline at end of file diff --git a/bhasai/utility/docker-compose.yaml b/bhasai/utility/docker-compose.yaml new file mode 100644 index 0000000..fb3d006 --- /dev/null +++ b/bhasai/utility/docker-compose.yaml @@ -0,0 +1,8 @@ +services: + fake: + environment: + DEFAULT_GITHUB_BRANCH: ${DEFAULT_GITHUB_BRANCH:-dev} + DEFAULT_IMAGE_TAG: ${DEFAULT_IMAGE_TAG:-dev} + org: ${org:-bharatsahaiyak} + AI_SERVICES: ${AI_SERVICES:-'[]'} + GRAFANA_DISCORD_WEBHOOK_URL: ${GRAFANA_DISCORD_WEBHOOK_URL:-https://} \ No newline at end of file diff --git a/sample.env b/sample.env index 8274043..5e972ce 100644 --- a/sample.env +++ b/sample.env @@ -1,93 +1,91 @@ -AI_TOOLS_EMBEDDING_BATCH_SIZE=100 -ALLOW_DEFAULT_OTP=false -CELERY_MAX_RETRIES=3 -CLICKHOUSE_DB=default -CLICKHOUSE_PASSWORD= -CLICKHOUSE_USER=default -DEFAULT_OTP=1234 -DEFAULT_OTP_USERS=[] -DEFAULT_RETRY_COUNT=3 -DEFAULT_TIMEOUT=30000 -DOCKER_REGISTRY_URL=ghcr.io -DOMAIN_NAME=localdev.me -DOMAIN_SCHEME=http -DOMAIN_SCHEME=https -ES_JAVA_OPTS=-Xms2048m -Xmx2048m -XX:InitiatingHeapOccupancyPercent=75 -Djava.awt.headless=true -FUSIONAUTH_APP_RUNTIME_MODE=development -GF_SECURITY_ADMIN_USER=admin -GF_SERVER_SERVE_FROM_SUB_PATH=false -LOKI_TABLE_INDEX_PERIOD=24h -NEXT_PUBLIC_DEBUG=false -NEXT_PUBLIC_SHOW_ONBOARDING=false -OTP_EXPIRY=30 -PARALLEL_FACTOR=2 -PDF_PARSER_MAX_RETRIES=3 -SPLIT=5 -TORCH_DEVICE=cuda -TRANSLATION_SERVICES=["azure", "bhasini"] -env=dev -ADMIN_API_KEY -ASR_SERVICE_PRIORITY -AZURE_SUBSCRIPTION_KEY -AZURE_TRANSLATE_KEY -BHASHINI_API_KEY -DEFAULT_USER_PASSWORD -DEPLOYER_APP_ENV -DEPLOYER_PWA_REPO -DEPLOYER_VERCEL_TOKEN -DISCORD_WEBHOOK_URL -DJANGO_SECRET_KEY -DOMAIN_NAME -DOMAIN_SCHEME -ENABLE_IMAGE_PARSING -ENABLE_SUMMARIZE -ENABLE_TRANSLATE -ENVIRONMENT_PASSWORD -ENVIRONMENT_USERNAME -FUSIONAUTH_ADMIN_EMAIL -FUSIONAUTH_ADMIN_PASSWORD -FUSIONAUTH_API_KEY -FUSIONAUTH_APP_ID -FUSIONAUTH_KEY -FUSIONAUTH_POSTGRES_PASSWORD -FUSIONAUTH_POSTGRES_ROOT_PASSWORD -FUSIONAUTH_POSTGRES_ROOT_USERNAME -FUSIONAUTH_POSTGRES_USERNAME -FUSIONAUTH_SALT -GF_SECURITY_ADMIN_PASSWORD -GITHUB_PAT -GIT_PASSWORD -GIT_USERNAME -GRAFANA_DISCORD_WEBHOOK_URL -GUPSHUP_OTP_TEMPLATE -GUPSHUP_PASSWORD -GUPSHUP_USERNAME -HUGGING_FACE_TOKEN -HUGGING_FACE_USERNAME -MINIO_ACCESS_KEY -MINIO_ROOT_PASSWORD -MINIO_ROOT_USER -MINIO_SECRET_KEY -NEXT_PUBLIC_BOT_ID -NEXT_PUBLIC_BOT_NAME -NEXT_PUBLIC_CLARITY_PROJECT_ID -NEXT_PUBLIC_ENTITY_DATASET_ID -NEXT_PUBLIC_GOOGLE_KEY -NEXT_PUBLIC_ORG_ID -NEXT_PUBLIC_USER_SERVICE_APP_ID -NEXT_PUBLIC_WEATHER_API -OPENAI_API_KEY -PDF_PARSER_SECRET_KEY -POSTGRES_PASSWORD -POSTGRES_USER -TLS_ENTRY -TRANSPORT_SOCKET_DEBUG -UCI_ADMIN_TOKEN -ULCA_API_KEY -ULCA_CONFIG_URL -ULCA_USER_ID -VAULT_PASSWORD -VAULT_USERNAME -YOUTUBE_DATA_API_KEY -env -org +AI_SERVICES=''[]'' +AI_TOOLS_EMBEDDING_BATCH_SIZE='100' +ALLOW_DEFAULT_OTP='false' +CELERY_MAX_RETRIES='3' +CLICKHOUSE_DB='default' +CLICKHOUSE_PASSWORD='' +CLICKHOUSE_USER='default' +DEFAULT_OTP='1234' +DEFAULT_OTP_USERS='[]' +DEFAULT_RETRY_COUNT='3' +DEFAULT_TIMEOUT='30000' +DOCKER_REGISTRY_URL='ghcr.io' +DOMAIN_NAME='localdev.me' +DOMAIN_SCHEME='http' +DOMAIN_SCHEME='https' +ES_JAVA_OPTS='-Xms2048m -Xmx2048m -XX:InitiatingHeapOccupancyPercent=75 -Djava.awt.headless=true' +FUSIONAUTH_APP_RUNTIME_MODE='development' +GF_SECURITY_ADMIN_USER='admin' +GF_SERVER_SERVE_FROM_SUB_PATH='false' +GRAFANA_DISCORD_WEBHOOK_URL='https://' +LOKI_TABLE_INDEX_PERIOD='24h' +NEXT_PUBLIC_DEBUG='false' +NEXT_PUBLIC_SHOW_ONBOARDING='false' +OTP_EXPIRY='30' +PARALLEL_FACTOR='2' +PDF_PARSER_MAX_RETRIES='3' +SPLIT='5' +TORCH_DEVICE='cuda' +TRANSLATION_SERVICES='["azure", "bhasini"]' +env='dev' +org='bharatsahaiyak' +ADMIN_API_KEY= +ASR_SERVICE_PRIORITY= +AZURE_SUBSCRIPTION_KEY= +AZURE_TRANSLATE_KEY= +BHASHINI_API_KEY= +DEFAULT_USER_PASSWORD= +DEPLOYER_APP_ENV= +DEPLOYER_PWA_REPO= +DEPLOYER_VERCEL_TOKEN= +DISCORD_WEBHOOK_URL= +DJANGO_SECRET_KEY= +ENABLE_IMAGE_PARSING= +ENABLE_SUMMARIZE= +ENABLE_TRANSLATE= +ENVIRONMENT_PASSWORD= +ENVIRONMENT_USERNAME= +FUSIONAUTH_ADMIN_EMAIL= +FUSIONAUTH_ADMIN_PASSWORD= +FUSIONAUTH_API_KEY= +FUSIONAUTH_APP_ID= +FUSIONAUTH_KEY= +FUSIONAUTH_POSTGRES_PASSWORD= +FUSIONAUTH_POSTGRES_ROOT_PASSWORD= +FUSIONAUTH_POSTGRES_ROOT_USERNAME= +FUSIONAUTH_POSTGRES_USERNAME= +FUSIONAUTH_SALT= +GF_SECURITY_ADMIN_PASSWORD= +GITHUB_PAT= +GIT_PASSWORD= +GIT_USERNAME= +GUPSHUP_OTP_TEMPLATE= +GUPSHUP_PASSWORD= +GUPSHUP_USERNAME= +HUGGING_FACE_TOKEN= +HUGGING_FACE_USERNAME= +MINIO_ACCESS_KEY= +MINIO_ROOT_PASSWORD= +MINIO_ROOT_USER= +MINIO_SECRET_KEY= +NEXT_PUBLIC_BOT_ID= +NEXT_PUBLIC_BOT_NAME= +NEXT_PUBLIC_CLARITY_PROJECT_ID= +NEXT_PUBLIC_ENTITY_DATASET_ID= +NEXT_PUBLIC_GOOGLE_KEY= +NEXT_PUBLIC_ORG_ID= +NEXT_PUBLIC_USER_SERVICE_APP_ID= +NEXT_PUBLIC_WEATHER_API= +OPENAI_API_KEY= +PDF_PARSER_SECRET_KEY= +POSTGRES_PASSWORD= +POSTGRES_USER= +TLS_ENTRY= +TRANSPORT_SOCKET_DEBUG= +UCI_ADMIN_TOKEN= +ULCA_API_KEY= +ULCA_CONFIG_URL= +ULCA_USER_ID= +VAULT_PASSWORD= +VAULT_USERNAME= +YOUTUBE_DATA_API_KEY= diff --git a/scripts/generate-sample-env.sh b/scripts/generate-sample-env.sh index d9312cb..8ec2f42 100755 --- a/scripts/generate-sample-env.sh +++ b/scripts/generate-sample-env.sh @@ -44,4 +44,8 @@ if [[ -s "$OUTPUT_FILE" ]]; then echo "Environment variables have been extracted to $OUTPUT_FILE" else echo "No environment variables found in the specified files." -fi \ No newline at end of file +fi + +sed -E -i "s/^([A-Za-z0-9_]+)=([^\n]*)$/\1='\2'/; s/^([A-Za-z0-9_]+)$/\1=/" $OUTPUT_FILE + +awk -F= '!seen[$1]++ || $2 != ""' $OUTPUT_FILE > temp_file && mv temp_file $OUTPUT_FILE From 5bfdfe2e96d24e7d68b9294896134e6b76a016cd Mon Sep 17 00:00:00 2001 From: Karun Agarwal <113603846+singhalkarun@users.noreply.github.com> Date: Mon, 12 Aug 2024 11:41:27 +0530 Subject: [PATCH 20/28] fix typo in install command of yq --- readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.md b/readme.md index 8af4ca3..0bcf414 100644 --- a/readme.md +++ b/readme.md @@ -29,7 +29,7 @@ In general every project requires observability, ci/cd pipelines, environment ma 3. Allow public inbound traffic on port 80 and Port 443 on the above VM (if you want to expose service publicly) 4. Allow public inbound traffic on port 9000 (if you want to expose deployment webhook publicly) 5. Run `sudo apt-get install build-essential` to install essential packages -6. Run `sudo wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/bin/yq &&\ +6. Run `sudo wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/bin/yq && sudo chmod +x /usr/bin/yq` to install [yq](https://github.com/mikefarah/yq) ## Setting up services on VM From 53aa0f9522273960bd88e639c27430caf69e4e0e Mon Sep 17 00:00:00 2001 From: Srijan Srivastava Date: Sat, 24 Aug 2024 23:08:53 +0530 Subject: [PATCH 21/28] feat: updated sample.env with domain prefix for document service --- bhasai/document_service/sample.env | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bhasai/document_service/sample.env b/bhasai/document_service/sample.env index 40c8da8..3dbba5a 100644 --- a/bhasai/document_service/sample.env +++ b/bhasai/document_service/sample.env @@ -4,4 +4,5 @@ POSTGRES_PASSWORD= MINIO_ACCESS_KEY= MINIO_SECRET_KEY= OPENAI_API_KEY= -AI_TOOLS_EMBEDDING_BATCH_SIZE= \ No newline at end of file +AI_TOOLS_EMBEDDING_BATCH_SIZE= +DOMAIN_NAME= From 5921ac74e145331dfb5ac25bbe05760be12f1b84 Mon Sep 17 00:00:00 2001 From: Srijan Srivastava Date: Sat, 24 Aug 2024 23:13:33 +0530 Subject: [PATCH 22/28] Update sample.env with VERCEL_ENV_CONFIG in service deployer --- bhasai/deployer/sample.env | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bhasai/deployer/sample.env b/bhasai/deployer/sample.env index 89f65bd..a54ee01 100644 --- a/bhasai/deployer/sample.env +++ b/bhasai/deployer/sample.env @@ -7,4 +7,5 @@ DEPLOYER_APP_ENV= DOMAIN_SCHEME= DOMAIN_NAME= MINIO_ACCESS_KEY= -MINIO_SECRET_KEY= \ No newline at end of file +MINIO_SECRET_KEY= +VERCEL_ENV_CONFIG= From db9b27e09b93ee65610fe68ae09beab7d93577b5 Mon Sep 17 00:00:00 2001 From: Karun Agarwal <113603846+singhalkarun@users.noreply.github.com> Date: Tue, 27 Aug 2024 11:22:24 +0530 Subject: [PATCH 23/28] udpate volume name to vault in bhasai vault service --- bhasai/vault/docker-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bhasai/vault/docker-compose.yaml b/bhasai/vault/docker-compose.yaml index 99592c2..87561b7 100644 --- a/bhasai/vault/docker-compose.yaml +++ b/bhasai/vault/docker-compose.yaml @@ -11,7 +11,7 @@ services: VAULT_USERNAME: ${VAULT_USERNAME:?VAULT_USERNAME is not set} VAULT_PASSWORD: ${VAULT_PASSWORD:?VAULT_PASSWORD is not set} volumes: - - environment:/vault/file + - vault:/vault/file - ./vault.json:/vault/config/vault.json - ./admin-policy.hcl:/vault/policies/admin-policy.hcl - ./vault-init.sh:/vault-init.sh From 4f291b29086d602fd9cbd1975046dac821df563b Mon Sep 17 00:00:00 2001 From: Karun Agarwal <113603846+singhalkarun@users.noreply.github.com> Date: Tue, 27 Aug 2024 11:25:56 +0530 Subject: [PATCH 24/28] update the volume name in bhasai vault service in the volumes section --- bhasai/vault/docker-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bhasai/vault/docker-compose.yaml b/bhasai/vault/docker-compose.yaml index 87561b7..bed34f7 100644 --- a/bhasai/vault/docker-compose.yaml +++ b/bhasai/vault/docker-compose.yaml @@ -1,5 +1,5 @@ volumes: - environment: + vault: services: vault: From 105b666483883788180af78cf42929195efeaa3e Mon Sep 17 00:00:00 2001 From: Karun Agarwal <113603846+singhalkarun@users.noreply.github.com> Date: Wed, 28 Aug 2024 12:47:50 +0530 Subject: [PATCH 25/28] remove consul from bhasai vault --- bhasai/vault/Dockerfile | 4 ---- bhasai/vault/consul-init.sh | 32 -------------------------------- bhasai/vault/docker-compose.yaml | 4 +--- bhasai/vault/env.tpl | 2 -- bhasai/vault/vault-init.sh | 2 -- 5 files changed, 1 insertion(+), 43 deletions(-) delete mode 100644 bhasai/vault/Dockerfile delete mode 100755 bhasai/vault/consul-init.sh delete mode 100644 bhasai/vault/env.tpl diff --git a/bhasai/vault/Dockerfile b/bhasai/vault/Dockerfile deleted file mode 100644 index 41acb0e..0000000 --- a/bhasai/vault/Dockerfile +++ /dev/null @@ -1,4 +0,0 @@ -FROM hashicorp/consul-template:0.37.2 as consul-template -FROM vault:1.13.3 -RUN apk add bash jq -COPY --from=consul-template /bin/consul-template /bin/consul-template \ No newline at end of file diff --git a/bhasai/vault/consul-init.sh b/bhasai/vault/consul-init.sh deleted file mode 100755 index dc79205..0000000 --- a/bhasai/vault/consul-init.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash - -set -e -pushd /bhasai - -export VAULT_ADDR="http://127.0.0.1:8200" - -# Authenticate using username and password -vault login -method=userpass username=$VAULT_USERNAME password=$VAULT_PASSWORD - -export VAULT_TOKEN=$(vault print token) - -export IFS=$'\n' -if ! vault kv get kv/env >/dev/null 2>&1; then - vault kv put kv/env \ - $(grep -v '^#' .env | grep -e '[^[:space:]]' | \ - sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' | \ - awk -F '=' '{gsub(/^["'\'']|["'\'']$/, "", $2); data[$1]=$2} END {for (key in data) print key "=" data[key]}') -# create a backup of the original .env -cp .env .env.bak -else - # Backup the last file created by Consul - if [ -f env.tmp ]; then - mv env.tmp .env.back - fi -fi - - -consul-template \ - -vault-default-lease-duration=10s \ - -vault-renew-token=false \ - -template "env.tpl:env.tmp:bash -c 'cat /bhasai/env.tmp > /bhasai/.env; echo .env updated '" diff --git a/bhasai/vault/docker-compose.yaml b/bhasai/vault/docker-compose.yaml index bed34f7..9e43703 100644 --- a/bhasai/vault/docker-compose.yaml +++ b/bhasai/vault/docker-compose.yaml @@ -3,9 +3,7 @@ volumes: services: vault: - build: - context: . - dockerfile: Dockerfile + image: vault:1.13.3 restart: always environment: VAULT_USERNAME: ${VAULT_USERNAME:?VAULT_USERNAME is not set} diff --git a/bhasai/vault/env.tpl b/bhasai/vault/env.tpl deleted file mode 100644 index fe7e39b..0000000 --- a/bhasai/vault/env.tpl +++ /dev/null @@ -1,2 +0,0 @@ -{{ with secret "kv/env" }} {{ range $k, $v := .Data.data }} -{{ printf "%s='%s'" $k $v | trimSpace }}{{ end }}{{ end }} \ No newline at end of file diff --git a/bhasai/vault/vault-init.sh b/bhasai/vault/vault-init.sh index 1f96f79..f268257 100644 --- a/bhasai/vault/vault-init.sh +++ b/bhasai/vault/vault-init.sh @@ -74,6 +74,4 @@ else echo "User $VAULT_USERNAME already exists." fi -/bhasai/consul-init.sh & - fg %1 From a1bb23363386d9f3b3e7274b7bc0a4b7cdb46ee9 Mon Sep 17 00:00:00 2001 From: Karun Agarwal <113603846+singhalkarun@users.noreply.github.com> Date: Wed, 28 Aug 2024 12:56:20 +0530 Subject: [PATCH 26/28] bhasai vault hotfix (add bash jq) --- bhasai/vault/Dockerfile | 2 ++ bhasai/vault/docker-compose.yaml | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 bhasai/vault/Dockerfile diff --git a/bhasai/vault/Dockerfile b/bhasai/vault/Dockerfile new file mode 100644 index 0000000..07b9f34 --- /dev/null +++ b/bhasai/vault/Dockerfile @@ -0,0 +1,2 @@ +FROM vault:1.13.3 +RUN apk add bash jq \ No newline at end of file diff --git a/bhasai/vault/docker-compose.yaml b/bhasai/vault/docker-compose.yaml index 9e43703..bed34f7 100644 --- a/bhasai/vault/docker-compose.yaml +++ b/bhasai/vault/docker-compose.yaml @@ -3,7 +3,9 @@ volumes: services: vault: - image: vault:1.13.3 + build: + context: . + dockerfile: Dockerfile restart: always environment: VAULT_USERNAME: ${VAULT_USERNAME:?VAULT_USERNAME is not set} From 0c6e4bb1e68711e938130960f939f2d9c5de5ae9 Mon Sep 17 00:00:00 2001 From: gauravjisri Date: Wed, 28 Aug 2024 17:59:28 +0530 Subject: [PATCH 27/28] added docs/no-internet-deployment.md --- docs/no-internet-deployment.md | 135 +++++++++++++++++++++++++++++++++ 1 file changed, 135 insertions(+) create mode 100644 docs/no-internet-deployment.md diff --git a/docs/no-internet-deployment.md b/docs/no-internet-deployment.md new file mode 100644 index 0000000..e8c8532 --- /dev/null +++ b/docs/no-internet-deployment.md @@ -0,0 +1,135 @@ +## Docker Package Installation + +### Overview : +- **Download**: Use a machine with internet access to download Docker and its dependencies. +- **Transfer**: Move the files to the server using `scp`. +- **Install**: Use `dpkg` on the server to install the packages. + +### Step 1: Download the Packages on a Machine with Internet Access + +1. **Set up Docker's apt repository:** + + On a machine with internet access, update your package list and set up Docker's apt repository: + ```bash + sudo apt-get update + sudo apt-get install -y ca-certificates curl + sudo install -m 0755 -d /etc/apt/keyrings + sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc + sudo chmod a+r /etc/apt/keyrings/docker.asc + ``` + +2. **Add Docker's repository to your apt sources:** + + ```bash + echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \ + $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \ + sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + sudo apt-get update + ``` + +3. **Download the Docker packages and their dependencies:** + + ```bash + mkdir -p ~/docker-packages + cd ~/docker-packages + sudo apt-get download docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin + ``` + + This will download all the necessary `.deb` files into the `~/docker-packages` directory. + +4. **Compress the folder containing the `.deb` files for easy transfer:** + + ```bash + tar -czvf docker-packages.tar.gz -C ~/ docker-packages + ``` + +### Step 2: Transfer the Packages to the Server + +1. **Use `scp` to transfer the compressed file to your server:** + + Replace `@:~/path/to/destination` with your server details: + ```bash + scp -i "/path/to/key" docker-packages.tar.gz @:~/path/to/destination + ``` + +2. **SSH into your server and navigate to the destination directory:** + + ```bash + ssh -i "/path/to/key" @ + cd ~/path/to/destination + ``` + +3. **Extract the compressed file:** + + ```bash + tar -xzvf docker-packages.tar.gz + ``` + +### Step 3: Install the Docker Packages on the Server + +1. **Navigate to the directory containing the `.deb` files:** + + ```bash + cd docker-packages + ``` + +2. **Install all the packages using `dpkg`:** + + ```bash + sudo dpkg -i *.deb + ``` + +3. **Fix any missing dependencies:** (Requires Internet) + + ```bash + sudo apt-get install -f + ``` + + This will install any missing dependencies from the `.deb` files that are already present in the directory. + + +## How to Use a Docker Image Offline. + +## Overview : +To use a Docker image offline, pull the `hello-world` image on a connected machine and save it as `hello-world.tar`. Transfer this file to the offline server, load it with `docker load -i hello-world.tar`, and then run it using `docker run hello-world`. + +### 1. Pre-download the `hello-world` Image + +On a machine with internet access, pull the `hello-world` Docker image: +```bash +docker pull hello-world +``` + +### 2. Save the Image + +Save the pulled `hello-world` image as a tar file: +```bash +docker save -o hello-world.tar hello-world +``` + +### 3. Transfer the Image to the Offline Server + +```bash +scp hello-world.tar user@your-server:/path/to/save +``` + +### 4. Load the Image on the Offline Server + +```bash +docker load -i /path/to/save/hello-world.tar +``` + +### 5. Verify the Image + +```bash +docker images +``` + +You should see the `hello-world` image listed. + +### 6. Use the Image + +```bash +docker run hello-world +``` \ No newline at end of file From bb408a1732dcb014c79569a24b394b88ae15e10b Mon Sep 17 00:00:00 2001 From: gauravjisri Date: Mon, 2 Sep 2024 10:50:59 +0530 Subject: [PATCH 28/28] modified docs/no-internet-deployment.md --- docs/no-internet-deployment.md | 295 ++++++++++++++++++++++++--------- 1 file changed, 217 insertions(+), 78 deletions(-) diff --git a/docs/no-internet-deployment.md b/docs/no-internet-deployment.md index e8c8532..16189fd 100644 --- a/docs/no-internet-deployment.md +++ b/docs/no-internet-deployment.md @@ -1,25 +1,83 @@ +## For installing yq : + +On machine with internet access(say Machine1): + +1. `sudo wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64` +2. `scp -i "key" yq_linux_amd64 ubuntu@:/tmp/yq_linux_amd64` + +On machine with no internet access(say Machine2): + +1. `sudo mv /tmp/yq_linux_amd64 /usr/bin/yq` +2. `sudo chmod +x /usr/bin/yq` +3. `yq --version` + +## For jq installation: (Not tested yet): +1. `sudo wget https://github.com/jqlang/jq/releases/download/jq-1.7.1/jq-linux-amd64` +2. `cp -i "key" jq_linux_amd64 ubuntu@:/tmp/jq_linux_amd64` + +On machine with no internet access(say Machine2): +1. `sudo mv /tmp/jq_linux_amd64 /usr/bin/jq` +2. `sudo chmod +x /usr/bin/jq` +3. `jq --version` + +## Installing build-essential : + +**Step 1: Download the Packages on a Machine with Internet Access** +1. On a machine with internet access, update your package list: `sudo apt-get update` +2. Download the `build-essential` package and its dependencies using `apt-get` with the `--download-only` option: + `sudo apt-get install --download-only build-essential + ` + This will download all the necessary `.deb` files to your system's cache (usually in `/var/cache/apt/archives`). +3. Collect the downloaded `.deb` files: + * `mkdir -p ~/build-essential-packages` + * `sudo cp /var/cache/apt/archives/*.deb ~/build-essential-packages/` + This will copy all the `.deb` files into the `~/build-essential-packages` directory. +4. Compress the folder containing the `.deb` files to make it easier to transfer: + `tar -czvf build-essential-packages.tar.gz -C ~/build-essential-packages . + ` + +**Step 2: Transfer the Packages to the Server** +1. Use `scp` to transfer the compressed file to your server (replace `user@server_ip:/path/to/destination` with your server details): + `scp build-essential-packages.tar.gz user@server_ip:/path/to/destination` +2. SSH into your server and navigate to the destination directory: + * `ssh user@server_ip` + * `cd /path/to/destination` +3. Extract the compressed file: + `tar -xzvf build-essential-packages.tar.gz` + +**Step 3: Install the Packages on the Server** +1. Navigate to the directory containing the `.deb` files: + `cd build-essential-packages` +2. Install all the packages using `dpkg`: + `sudo dpkg -i *.deb` +3. If there are any missing dependencies, fix them by running: + `sudo apt-get install -f` + +If your system does not have internet access and you run apt-get install -f, it might not be able to fix the broken dependencies unless you have already downloaded the necessary packages and dependencies to your local cache or have them available in a local repository. In such cases, you would need to manually download the required packages and install them. + +4. Add the current user in docker group: +`usermod -aG docker ${USER}` + ## Docker Package Installation -### Overview : +**Overview :** - **Download**: Use a machine with internet access to download Docker and its dependencies. - **Transfer**: Move the files to the server using `scp`. - **Install**: Use `dpkg` on the server to install the packages. -### Step 1: Download the Packages on a Machine with Internet Access +### Step 1: Download the Packages on a Machine with Internet Access** 1. **Set up Docker's apt repository:** On a machine with internet access, update your package list and set up Docker's apt repository: - ```bash - sudo apt-get update - sudo apt-get install -y ca-certificates curl - sudo install -m 0755 -d /etc/apt/keyrings - sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc - sudo chmod a+r /etc/apt/keyrings/docker.asc - ``` -2. **Add Docker's repository to your apt sources:** + `sudo apt-get update` + `sudo apt-get install -y ca-certificates curl` + `sudo install -m 0755 -d /etc/apt/keyrings` + `sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc` + `sudo chmod a+r /etc/apt/keyrings/docker.asc` +2. **Add Docker's repository to your apt sources:** ```bash echo \ "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \ @@ -27,109 +85,190 @@ sudo tee /etc/apt/sources.list.d/docker.list > /dev/null sudo apt-get update ``` - 3. **Download the Docker packages and their dependencies:** + `mkdir -p ~/docker-packages` + `cd ~/docker-packages` + `sudo apt-get download docker-ce docker-ce-cli containerd.io` + `docker-buildx-plugin` + `docker-compose-plugin` + + This will download all the necessary `.deb` files into the `~/docker-packages` directory. +4. **Compress the folder containing the `.deb` files for easy transfer:** + `tar -czvf docker-packages.tar.gz -C ~/ docker-packages` - ```bash - mkdir -p ~/docker-packages - cd ~/docker-packages - sudo apt-get download docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin - ``` +### Step 2: Transfer the Packages to the Server - This will download all the necessary `.deb` files into the `~/docker-packages` directory. +1. **Use `scp` to transfer the compressed file to your server:** + Replace `@:~/path/to/destination` with your server details: + `scp -i "/path/to/key" docker-packages.tar.gz @:~/path/to/destination` +2. **SSH into your server and navigate to the destination directory:** + `ssh -i "/path/to/key" @` + `cd ~/path/to/destination` +3. **Extract the compressed file:** + `tar -xzvf docker-packages.tar.gz` -4. **Compress the folder containing the `.deb` files for easy transfer:** +### Step 3: Install the Docker Packages on the Server - ```bash - tar -czvf docker-packages.tar.gz -C ~/ docker-packages - ``` +1. **Navigate to the directory containing the `.deb` files:** + `cd docker-packages` +2. **Install all the packages using `dpkg`:** + `sudo dpkg -i *.deb` + 3. **Fix any missing dependencies:** (Requires Internet) + `sudo apt-get install -f` -### Step 2: Transfer the Packages to the Server + This will install any missing dependencies from the `.deb` files that are already present in the directory. -1. **Use `scp` to transfer the compressed file to your server:** - Replace `@:~/path/to/destination` with your server details: - ```bash - scp -i "/path/to/key" docker-packages.tar.gz @:~/path/to/destination - ``` +## How to Use a Docker Image Offline. -2. **SSH into your server and navigate to the destination directory:** +## Overview : +To use a Docker image offline, pull the `hello-world` image on a connected machine and save it as `hello-world.tar`. Transfer this file to the offline server, load it with `docker load -i hello-world.tar`, and then run it using `docker run hello-world`. - ```bash - ssh -i "/path/to/key" @ - cd ~/path/to/destination - ``` +**1. Pre-download the `hello-world` Image** +* On a machine with internet access, pull the `hello-world` Docker image: +`docker pull hello-world` -3. **Extract the compressed file:** +**2. Save the Image** +Save the pulled `hello-world` image as a tar file: +* `docker save -o hello-world.tar hello-world` - ```bash - tar -xzvf docker-packages.tar.gz - ``` +**3. Transfer the Image to the Offline Server** +* `scp hello-world.tar user@your-server:/path/to/save` -### Step 3: Install the Docker Packages on the Server +**4. Load the Image on the Offline Server** +* `docker load -i /path/to/save/hello-world.tar` -1. **Navigate to the directory containing the `.deb` files:** +**5. Verify the Image** +* `docker images` +You should see the `hello-world` image listed. - ```bash - cd docker-packages - ``` +**6. Use the Image** +* `docker run hello-world` -2. **Install all the packages using `dpkg`:** - ```bash - sudo dpkg -i *.deb - ``` +## Installing GPU packages: +To download all the necessary NVIDIA CUDA drivers, toolkit, and Docker-related packages into a separate folder named `nvidia_packages` on the machine with internet access, you can use the following script: + +### Step 1: Download the Packages to `nvidia_packages` Folder + +```bash +#!/bin/bash + +# Create a directory to store all downloaded packages +mkdir -p ~/nvidia_packages +cd ~/nvidia_packages + +# Check if an NVIDIA GPU is present +if lspci | grep -i nvidia &>/dev/null; then + echo "NVIDIA GPU found." + echo "Downloading NVIDIA CUDA drivers and Toolkit" + export distro="$(lsb_release -is | tr '[:upper:]' '[:lower:]')$(lsb_release -rs | tr -d '.')" + export arch=$(uname -m) + + # Download the CUDA keyring + wget https://developer.download.nvidia.com/compute/cuda/repos/$distro/$arch/cuda-keyring_1.1-1_all.deb + + # Add NVIDIA repo and download CUDA drivers + sudo dpkg -i cuda-keyring_1.1-1_all.deb + sudo apt-get -qq update + apt-get download $(apt-cache depends cuda-drivers | grep Depends | sed "s/.*ends:\ //" | tr '\n' ' ') + + echo "Downloading NVIDIA container toolkit" + curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | sudo gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg + curl -s -L https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list | \ + sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' | \ + sudo tee /etc/apt/sources.list.d/nvidia-container-toolkit.list + + sudo apt-get -qq update + apt-get download nvidia-container-toolkit nvidia-docker2 + + echo "Downloading Docker-related packages" + apt-get download jq docker.io + + echo "All NVIDIA and Docker packages downloaded to ~/nvidia_packages" +else + echo "No NVIDIA GPU found. Skipping NVIDIA package download." +fi + +``` + +### Step 2: Transfer the `nvidia_packages` Folder + +1. **Transfer the `nvidia_packages` Folder via `scp`:** + + + `scp -i "key" -r ~/nvidia_packages user@offline-machine:/path/to/destination` + +### Step 3: Install the Packages on the Offline Machine -3. **Fix any missing dependencies:** (Requires Internet) +1. **Install the Packages on the Offline Machine:** + + On the offline machine, navigate to the directory where you transferred the `nvidia_packages` folder and install all the `.deb` packages: + + `cd /path/to/destination/nvidia_packages` + `sudo dpkg -i *.deb` + `sudo apt-get install -f` + +2. **Restart the machine** + + +## Setting up Webhook : + +Given that you already have the necessary template files (`hooks.json.template`, `webhook.service.template`) and the `.env` file on the offline machine, the primary task is to install the `webhook` binary and then run your setup script. Here’s how to proceed: + +### Step 1: Prepare the `webhook` Binary on the Internet-Connected Machine + +1. **Download the Webhook Binary:** + + On the machine with internet access, download the `webhook` binary: ```bash - sudo apt-get install -f + mkdir -p ~/webhook_setup + cd ~/webhook_setup + curl -sSL https://github.com/adnanh/webhook/releases/download/2.8.1/webhook-linux-amd64.tar.gz -o webhook-linux-amd64.tar.gz + tar -zxvf webhook-linux-amd64.tar.gz --strip-components=1 --directory . ``` - This will install any missing dependencies from the `.deb` files that are already present in the directory. +2. **Transfer the Binary to the Offline Machine:** + Transfer the `webhook` binary to the offline machine using `scp`: -## How to Use a Docker Image Offline. + ```bash + scp ~/webhook_setup/webhook user@offline-machine:/path/to/destination + ``` -## Overview : -To use a Docker image offline, pull the `hello-world` image on a connected machine and save it as `hello-world.tar`. Transfer this file to the offline server, load it with `docker load -i hello-world.tar`, and then run it using `docker run hello-world`. +### Step 2: Install the `webhook` Binary on the Offline Machine -### 1. Pre-download the `hello-world` Image +1. **Move the Binary to `/usr/local/bin`:** -On a machine with internet access, pull the `hello-world` Docker image: -```bash -docker pull hello-world -``` + On the offline machine, move the `webhook` binary to `/usr/local/bin` and make it executable: -### 2. Save the Image + ```bash + sudo mv /path/to/destination/webhook /usr/local/bin/ + sudo chmod +x /usr/local/bin/webhook + ``` -Save the pulled `hello-world` image as a tar file: -```bash -docker save -o hello-world.tar hello-world -``` +### Step 3: Run the Setup Script on the Offline Machine -### 3. Transfer the Image to the Offline Server +1. **Ensure the `.env` File is Correct:** -```bash -scp hello-world.tar user@your-server:/path/to/save -``` + Verify that your `.env` file has the correct values for `WEBHOOK_PASSWORD`, `WEBHOOK_USER`, and `WEBHOOK_GROUP`. -### 4. Load the Image on the Offline Server +2. **Run the Setup Script:** -```bash -docker load -i /path/to/save/hello-world.tar -``` + Navigate to the directory where your setup script is located and run it: -### 5. Verify the Image + ```bash + cd /path/to/setup_script + ./setup_webhook.sh + ``` -```bash -docker images -``` +### Step 4: Verify the Setup -You should see the `hello-world` image listed. +1. **Check the Webhook Service Status:** -### 6. Use the Image + After running the setup script, ensure that the webhook service is running: -```bash -docker run hello-world -``` \ No newline at end of file + ```bash + sudo systemctl status webhook.service + ``` \ No newline at end of file