working configuration

2026-03-22 19:19:26 +01:00
parent c1a4025697
commit 5b83e1f492
63 changed files with 62182 additions and 19 deletions
--- a/productivity/overleaf/services/history-v1/docker-compose.ci.yml
+++ b/productivity/overleaf/services/history-v1/docker-compose.ci.yml
@@ -0,0 +1,278 @@
+# This file was auto-generated, do not edit it directly.
+# Instead run bin/update_build_scripts from
+# https://github.com/overleaf/internal/
+
+volumes:
+  minio-certs:
+
+services:
+  test_unit:
+    image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
+    user: node
+    volumes:
+      - ./reports:/overleaf/services/history-v1/reports
+      - ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it
+    entrypoint: /overleaf/bin/shared/wait_for_it mongo:27017 --timeout=60 --
+    command: npm run test:unit:_run
+    environment:
+      CI:
+      MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf
+      NODE_ENV: test
+      NODE_OPTIONS: "--unhandled-rejections=strict"
+      REDIS_HOST: redis_test
+      QUEUES_REDIS_HOST: redis_test
+      HISTORY_REDIS_HOST: redis_test
+      ANALYTICS_QUEUES_REDIS_HOST: redis_test
+      ANALYTICS_QUEUES_BQ_SINK_REDIS_HOST: redis_test
+    depends_on:
+      mongo:
+        condition: service_started
+      redis_test:
+        condition: service_healthy
+
+  test_acceptance:
+    build: .
+    image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
+    environment:
+      CI:
+      RETRIES:
+      REDIS_HOST: redis_test
+      QUEUES_REDIS_HOST: redis_test
+      HISTORY_REDIS_HOST: redis_test
+      ANALYTICS_QUEUES_REDIS_HOST: redis_test
+      ANALYTICS_QUEUES_BQ_SINK_REDIS_HOST: redis_test
+      MONGO_HOST: mongo
+      POSTGRES_HOST: postgres
+      AWS_S3_ENDPOINT: https://minio:9000
+      AWS_S3_PATH_STYLE: "true"
+      DELETE_OBJECTS_MD5_FALLBACK: true
+      AWS_ACCESS_KEY_ID: OVERLEAF_HISTORY_S3_ACCESS_KEY_ID
+      AWS_SECRET_ACCESS_KEY: OVERLEAF_HISTORY_S3_SECRET_ACCESS_KEY
+      MINIO_ROOT_USER: MINIO_ROOT_USER
+      MINIO_ROOT_PASSWORD: MINIO_ROOT_PASSWORD
+      GCS_API_ENDPOINT: http://gcs:9090
+      GCS_PROJECT_ID: fake
+      STORAGE_EMULATOR_HOST: http://gcs:9090/storage/v1
+      MOCHA_GREP: ${MOCHA_GREP}
+      NODE_ENV: test
+      NODE_OPTIONS: "--unhandled-rejections=strict"
+    volumes:
+      - ./reports:/overleaf/services/history-v1/reports
+      - minio-certs:/certs
+      - ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it
+    depends_on:
+      mongo:
+        condition: service_started
+      redis_test:
+        condition: service_healthy
+      postgres:
+        condition: service_healthy
+      certs:
+        condition: service_completed_successfully
+      minio:
+        condition: service_started
+      minio_setup:
+        condition: service_completed_successfully
+      gcs:
+        condition: service_healthy
+    user: node
+    entrypoint: /overleaf/bin/shared/wait_for_it mongo:27017 --timeout=60 --
+    command: npm run test:acceptance
+
+  tar:
+    build: .
+    image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
+    volumes:
+      - ./:/tmp/build/
+    command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs .
+    user: root
+
+  redis_test:
+    image: redis:7.4.3
+    healthcheck:
+      test: ping="$$(redis-cli ping)" && [ "$$ping" = 'PONG' ]
+      interval: 1s
+      retries: 20
+
+  mongo:
+    image: mongo:8.0.11
+    command: --replSet overleaf --notablescan
+    volumes:
+      - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
+      - ../../bin/shared/mongodb-docker-entrypoint-wait.sh:/mongodb-docker-entrypoint-wait.sh
+    entrypoint: /mongodb-docker-entrypoint-wait.sh
+    environment:
+      MONGO_INITDB_DATABASE: sharelatex
+    extra_hosts:
+      # Required when using the automatic database setup for initializing the
+      # replica set. This override is not needed when running the setup after
+      # starting up mongo.
+      - mongo:127.0.0.1
+
+  postgres:
+    image: postgres:10
+    environment:
+      POSTGRES_USER: overleaf
+      POSTGRES_PASSWORD: overleaf
+      POSTGRES_DB: overleaf-history-v1-test
+    volumes:
+      - ./test/acceptance/pg-init/:/docker-entrypoint-initdb.d/
+    healthcheck:
+      test: pg_isready --quiet
+      interval: 1s
+      retries: 20
+
+  certs:
+    build:
+      dockerfile_inline: |
+        FROM node:24.13.0
+        RUN wget -O /certgen "https://github.com/minio/certgen/releases/download/v1.3.0/certgen-linux-"`dpkg --print-architecture`
+        RUN chmod +x /certgen
+    volumes:
+      - minio-certs:/certs
+    working_dir: /certs
+    entrypoint: sh
+    command:
+      - "-cex"
+      - |
+        if [ ! -f private.key ] || [ ! -f public.crt ]; then
+          /certgen -host minio
+        fi
+        if ! openssl x509 -checkend 864000 -noout -in public.crt > /dev/null; then
+          /certgen -host minio
+        fi
+
+  minio:
+    image: minio/minio:RELEASE.2024-10-13T13-34-11Z
+    command: server /data
+    volumes:
+      - minio-certs:/root/.minio/certs
+    environment:
+      MINIO_ROOT_USER: MINIO_ROOT_USER
+      MINIO_ROOT_PASSWORD: MINIO_ROOT_PASSWORD
+    depends_on:
+      certs:
+        condition: service_completed_successfully
+
+  minio_setup:
+    depends_on:
+      certs:
+        condition: service_completed_successfully
+      minio:
+        condition: service_started
+    image: minio/mc:RELEASE.2024-10-08T09-37-26Z
+    volumes:
+      - minio-certs:/root/.mc/certs/CAs
+    entrypoint: sh
+    command:
+      - "-cex"
+      - |
+        sleep 1
+        mc alias set s3 https://minio:9000 MINIO_ROOT_USER MINIO_ROOT_PASSWORD \
+        || sleep 3 && \
+        mc alias set s3 https://minio:9000 MINIO_ROOT_USER MINIO_ROOT_PASSWORD \
+        || sleep 3 && \
+        mc alias set s3 https://minio:9000 MINIO_ROOT_USER MINIO_ROOT_PASSWORD \
+        || sleep 3 && \
+        mc alias set s3 https://minio:9000 MINIO_ROOT_USER MINIO_ROOT_PASSWORD
+        mc mb --ignore-existing s3/overleaf-test-history-chunks
+        mc mb --ignore-existing s3/overleaf-test-history-deks
+        mc mb --ignore-existing s3/overleaf-test-history-global-blobs
+        mc mb --ignore-existing s3/overleaf-test-history-project-blobs
+        mc admin user add s3 \
+          OVERLEAF_HISTORY_S3_ACCESS_KEY_ID \
+          OVERLEAF_HISTORY_S3_SECRET_ACCESS_KEY
+        echo '
+          {
+            "Version": "2012-10-17",
+            "Statement": [
+              {
+                "Effect": "Allow",
+                "Action": [
+                  "s3:ListBucket"
+                ],
+                "Resource": "arn:aws:s3:::overleaf-test-history-chunks"
+              },
+              {
+                "Effect": "Allow",
+                "Action": [
+                  "s3:PutObject",
+                  "s3:GetObject",
+                  "s3:DeleteObject",
+                  "s3:AbortMultipartUpload",
+                  "s3:ListMultipartUploadParts",
+                  "s3:ListBucketMultipartUploads"
+                ],
+                "Resource": "arn:aws:s3:::overleaf-test-history-chunks/*"
+              },
+              {
+                "Effect": "Allow",
+                "Action": [
+                  "s3:ListBucket"
+                ],
+                "Resource": "arn:aws:s3:::overleaf-test-history-deks"
+              },
+              {
+                "Effect": "Allow",
+                "Action": [
+                  "s3:PutObject",
+                  "s3:GetObject",
+                  "s3:DeleteObject",
+                  "s3:AbortMultipartUpload",
+                  "s3:ListMultipartUploadParts",
+                  "s3:ListBucketMultipartUploads"
+                ],
+                "Resource": "arn:aws:s3:::overleaf-test-history-deks/*"
+              },
+              {
+                "Effect": "Allow",
+                "Action": [
+                  "s3:ListBucket"
+                ],
+                "Resource": "arn:aws:s3:::overleaf-test-history-global-blobs"
+              },
+              {
+                "Effect": "Allow",
+                "Action": [
+                  "s3:PutObject",
+                  "s3:GetObject",
+                  "s3:DeleteObject",
+                  "s3:AbortMultipartUpload",
+                  "s3:ListMultipartUploadParts",
+                  "s3:ListBucketMultipartUploads"
+                ],
+                "Resource": "arn:aws:s3:::overleaf-test-history-global-blobs/*"
+              },
+              {
+                "Effect": "Allow",
+                "Action": [
+                  "s3:ListBucket"
+                ],
+                "Resource": "arn:aws:s3:::overleaf-test-history-project-blobs"
+              },
+              {
+                "Effect": "Allow",
+                "Action": [
+                  "s3:PutObject",
+                  "s3:GetObject",
+                  "s3:DeleteObject",
+                  "s3:AbortMultipartUpload",
+                  "s3:ListMultipartUploadParts",
+                  "s3:ListBucketMultipartUploads"
+                ],
+                "Resource": "arn:aws:s3:::overleaf-test-history-project-blobs/*"
+              }
+            ]
+          }' > policy-history.json
+
+        mc admin policy create s3 overleaf-history policy-history.json
+        mc admin policy attach s3 overleaf-history \
+          --user=OVERLEAF_HISTORY_S3_ACCESS_KEY_ID
+
+  gcs:
+    image: fsouza/fake-gcs-server:1.52.3
+    command: ["--port=9090", "--scheme=http", "--external-url=http://gcs:9090"]
+    healthcheck:
+      test: wget --quiet --output-document=/dev/null http://localhost:9090/storage/v1/b
+      interval: 1s
+      retries: 20