# Copied from mongosync/evergreen, with slight changes.

# Functions and variables which differ from mongosync/evergreen/functions.yml are
# marked as
#   either: "Added for mongodump_passthrough"
#   or:     "Changed for mongodump_passthrough"
#
# The tests are run by loading the mongosync repo as an evergreen module under src/mongosync.
# The sources for mongodump-suite-gen, mongodump-task-gen, js tests (with slight modifications
# for mongodump), and the dump_restore.py are all in the mongosync repo.
#
# Changes made to these in the mongosync repo will only be picked up when the _mongosync_pin
# is changed here to point at the new mongosync commit hash.

variables:
  _migration_verifier_pin: &_migration_verifier_pin "9431222bdc602e21b3cf2a64ee45d5eaceac6060"
  _resmoke_dir: &_resmoke_dir "src/resmoke"
  _src_dir: &src_dir src/mongosync
  #
  # WARNING:
  #
  # There is a delicate relationship between the mongodump_passthrough evergreen
  # configuration files, and the pinned mongosync which it fetches as an evergreen module.
  # When you update this pin, you may need to make other changes to get mongodump_passthrough
  # working again.
  #
  # And when you change the _mongosync_pin, you also need to update other pins
  # which get passed as environment variables to scripts in mongosync/evergreen/scripts.
  #
  _mongosync_pin: &_mongosync_pin "1f8541766d415c9de12aee6760926ba93ece45ae"

  f_expansions_write: &f_expansions_write
    command: expansions.write
    params:
      file: expansions.yml
      redacted: true

  # Changed for mongodump_passthrough
  f_repo_fetch: &f_repo_fetch
    command: git.get_project
    params:
      directory: src/github.com/mongodb/mongo-tools
      revisions:
        mongosync: *_mongosync_pin

  f_set_up_logger_credentials: &f_set_up_logger_credentials
    command: subprocess.exec
    params:
      binary: "./src/mongosync/evergreen/scripts/logger_credentials_setup.sh"
      add_expansions_to_env: true

  # Added for mongodump_passthrough
  f_link_to_mongo_tools: &f_link_to_mongo_tools
    command: shell.exec
    params:
      shell: bash
      script: |
        ln -s ${workdir}/src/github.com/mongodb/mongo-tools src

  f_resmoke_wheelhouse_setup: &f_resmoke_wheelhouse_setup
    command: subprocess.exec
    params:
      binary: "./src/mongosync/evergreen/scripts/resmoke_wheelhouse_setup.sh"
      add_expansions_to_env: true

  # Function used to download particular bits of resmoke and binaries uploaded previously.
  f_resmoke_with_binaries_fetch: &f_resmoke_with_binaries_fetch
    command: s3.get
    params:
      aws_key: ${aws_key}
      aws_secret: ${aws_secret}
      # Changed for mongodump_passthrough
      remote_file: mongo-tools/mongodump_passthrough/${build_variant}/${revision}/resmoke-with-binaries/${build_id}.tgz
      bucket: mciuploads
      extract_to: *_resmoke_dir

  f_resmoke_wheelhouse_fetch: &f_resmoke_wheelhouse_fetch
    command: s3.get
    params:
      aws_key: ${aws_key}
      aws_secret: ${aws_secret}
      # Changed for mongodump_passthrough
      remote_file: mongo-tools/mongodump_passthrough/${build_variant}/${revision}/resmoke-python-wheelhouse/${build_id}.tgz
      bucket: mciuploads
      extract_to: "."

  f_migration_verifier_binary_fetch: &f_migration_verifier_binary_fetch
    command: s3.get
    params:
      aws_key: ${aws_key}
      aws_secret: ${aws_secret}
      # Changed for mongodump_passthrough
      remote_file: mongo-tools/mongodump_passthrough/${mongosync_compile_build_variant}/${revision}/${version_id}/migration_verifier
      bucket: mciuploads
      local_file: src/mongosync/migration_verifier

  f_make_migration_verifier_binary_executable:
    &f_make_migration_verifier_binary_executable
    command: shell.exec
    params:
      shell: bash
      working_dir: src/mongosync
      script: |
        set -o errexit
        set -o pipefail
        set -o xtrace

        chmod +x migration_verifier

  # This fetches the contents copied from mongosync/dist
  # We keep the same function name as in mongosync evergreen, but here that
  # directory contains the mongodump and mongorestore binaries.
  f_mongosync_binary_fetch: &f_mongosync_binary_fetch
    command: s3.get
    params:
      aws_key: ${aws_key}
      aws_secret: ${aws_secret}
      # Changed for mongodump_passthrough
      remote_file: mongo-tools/mongodump_passthrough/${mongosync_compile_build_variant}/${revision}/${version_id}/${mongosync_binary_folder}/${version_id}.tgz
      bucket: mciuploads
      extract_to: "src/mongosync"

  f_generate_github_access_token: &f_generate_github_access_token
    command: github.generate_token
    params:
      owner: 10gen
      repo: mongo
      expansion_name: generated_token_mongo
      permissions:
        contents: read

  f_resmoke_jobs_determine: &f_resmoke_jobs_determine
    command: subprocess.exec
    params:
      binary: "./src/mongosync/evergreen/scripts/resmoke_jobs_determine.sh"
      add_expansions_to_env: true
      env:
        resmoke_jobs_factor: ${resmoke_jobs_factor|1}
        resmoke_jobs_max: ${resmoke_jobs_max|12}

  f_resmoke_jobs_expansion_update: &f_resmoke_jobs_expansion_update
    command: expansions.update
    params:
      ignore_missing_file: true
      file: src/resmoke/resmoke_jobs_expansion.yml

  f_resmoke_tests_execute: &f_resmoke_tests_execute
    command: subprocess.exec
    type: test
    params:
      binary: "./src/mongosync/evergreen/scripts/resmoke_tests_execute.sh"
      add_expansions_to_env: true
      env:
        resmoke_jobs: ${resmoke_jobs|1}
        should_shuffle: ${should_shuffle|true}

  f_infrastructure_failure_check: &f_infrastructure_failure_check
    # The existence of the "infrastructure_failure" file indicates this failure isn't
    # directly actionable. We use type=setup rather than type=system or type=test for this command
    # because we don't intend for any human to look at this failure.
    command: subprocess.exec
    type: setup
    params:
      binary: "./src/mongosync/evergreen/scripts/infrastructure_failure_check.sh"
      add_expansions_to_env: true

  f_set_up_archive_credentials: &f_set_up_archive_credentials
    command: subprocess.exec
    params:
      binary: "./src/mongosync/evergreen/scripts/archive_credentials_setup.sh"
      env:
        private_key_remote_bash_var: ${private_key_remote}
      add_expansions_to_env: true

  get_buildnumber: &get_buildnumber
    command: keyval.inc
    params:
      key: "${build_variant}_v70"
      destination: "builder_num"

functions:
  # Changed for mongodump_passthrough
  "fetch source and install go":
    # This clones mongo-tools and mongosync.
    - command: git.get_project
      params:
        directory: src/github.com/mongodb/mongo-tools
        revisions:
          mongosync: *_mongosync_pin
          migration-verifier: *_migration_verifier_pin
    # The mongo-tools repo is cloned into src/github.com/mongodb/mongo-tools, but
    # we also create a symbolic link at src/mongo-tools.
    - *f_link_to_mongo_tools
    # Check that mongo-tools and mongosync both got cloned to the right place.
    - command: shell.exec
      params:
        shell: bash
        script: |
          if [ ! -d "src/github.com/mongodb/mongo-tools" ]; then
            echo "ERROR: git.get_project did not clone mongo-tools as required."
            find . -name "mongo-tools"
            exit 1
          fi
          if [ ! -d "src/mongosync" ]; then
            echo "ERROR: git.get_project did not clone mongosync as required."
            echo "       Check that buildvariant ${build_variant} lists module mongosync."
            find . -name "mongosync"
            exit 1
          fi

    # Make an evergreen expansion file with dynamic values
    - command: subprocess.exec
      params:
        binary: "./src/mongosync/evergreen/scripts/generate_mongosync_expansion.sh"
        add_expansions_to_env: true
        env:
          mongosync_dir: "src/mongosync"
    # Load the expansion file to make an evergreen variable with the current unique version
    - command: expansions.update
      params:
        file: src/mongosync/mongosync_expansion.yml
    # Tasks/functions that use the mongosync sources should use the same
    # mise-installed tools as when running from mongo-tools evergreen.
    - command: shell.exec
      params:
        shell: bash
        working_dir: src/mongosync
        script: |
          set -o errexit
          set -o pipefail
          set -o xtrace

          ${PREPARE_SHELL}
          source "./etc/functions.sh"

          MISE_INSTALL_PATH='${workdir}/.local/bin/mise' retry sh ./etc/mise.run.sh
          mise settings experimental=true
          # We only install Go here in order to get the system to a state where we can run mage. If
          # more dev tools are needed, those tasks should run "mise install".
          mise install go
    - command: shell.exec
      params:
        shell: bash
        working_dir: src/mongosync
        script: |
          set -o errexit
          set -o pipefail
          set -o xtrace

          ${PREPARE_SHELL}

          mise exec -- go version
          mise exec -- go env

  # Added for mongodump_passthrough
  #
  # This builds the mongo-tools executables, then copies mongogump and mongorestore
  # into src/mongosync/dist. We use the same shell setup and go toolchain
  # as in mongo-tools/common.yml "run make target"
  "build mongodump and mongorestore":
    - command: shell.exec
      params:
        working_dir: src/github.com/mongodb/mongo-tools
        script: |
          ${_set_shell_env}
          ${_maybe_enable_devtoolset_7}
          PATH=$PATH:$HOME
          go run build.go -v build
          # Copy mongodump and mongorestore to mongosync/dist
          mkdir -p ${workdir}/src/mongosync/dist
          cp -p bin/mongodump bin/mongorestore ${workdir}/src/mongosync/dist

  f_repo_with_modules_fetch:
    - command: git.get_project
      params:
        directory: src/github.com/mongodb/mongo-tools
        revisions:
          mongosync: *_mongosync_pin
          migration-verifier: *_migration_verifier_pin
    # Check that mongo-tools and mongosync both got cloned to the right place.
    - command: shell.exec
      params:
        shell: bash
        script: |
          if [ ! -d "src/github.com/mongodb/mongo-tools" ]; then
            echo "ERROR: git.get_project did not clone mongo-tools as required."
            exit 1
          fi
          ln -s ${workdir}/src/github.com/mongodb/mongo-tools ${workdir}/src/mongo-tools
          if [ ! -d "src/mongosync" ]; then
            echo "ERROR: git.get_project did not clone mongosync as required."
            echo "       Check that buildvariant ${build_variant} lists module mongosync."
            exit 1
          fi

  # Under src/mongosync/resmoke we have separate directories "suite-config"
  # and "mongodump-suite-config". To run mongodump passthrough tests we delete
  # mongosync-related suites directories and copy the mongodump_suite-config.
  # This keeps the mongosync suite-config/*/loggers dirs.
  f_enable_mongodump_suite_config: &f_enable_mongodump_suite_config
    command: shell.exec
    params:
      shell: bash
      working_dir: src/mongosync/resmoke
      script: |
        set -o errexit
        set -o pipefail
        set -o xtrace

        rm -rf `find suite-config -name "suites"`
        cd mongodump-suite-config
        tar cvf ../suite-config/a.tar .
        cd ../suite-config
        tar xvf a.tar

  f_expansions_write:
    - *f_expansions_write

  f_generate_github_access_token:
    - *f_generate_github_access_token

  "configure evergreen api credentials": &f_evergreen_credentials_setup
    command: subprocess.exec
    params:
      # The evergreen project variables in mongo-tools are ${evg_key} and ${evg_user}.
      binary: "./src/mongosync/evergreen/scripts/evergreen_credentials_setup.sh"
      env:
        evergreen_api_key: ${evg_key}
        evergreen_api_user: ${evg_user}
      add_expansions_to_env: true

  # Function used to download Resmoke and compiled mongo* binaries from the server project.
  f_resmoke_and_binaries_download:
    - *f_evergreen_credentials_setup
    - command: subprocess.exec
      params:
        binary: "./src/mongosync/evergreen/scripts/resmoke_and_binaries_download.sh"
        add_expansions_to_env: true

  f_resmoke_wheelhouse_setup:
    - *f_resmoke_wheelhouse_setup

  f_resmoke_archive:
    - command: archive.targz_pack
      params:
        target: resmoke-with-binaries.tgz
        source_dir: "src/resmoke/${src_mongo_version}_${dst_mongo_version}"
        include:
          - "./buildscripts/**"
          - "./${src_mongo_version}/**"
          - "./${dst_mongo_version}/**"
          - "./mongo-for-shell/**"
          - "./etc/**"
          - "./jstests/**"
          - "./src/**"
          - "./.resmoke_mongo_version.yml"
          - "./.resmoke_mongo_release_values.yml"
    - command: s3.put
      params:
        aws_key: ${aws_key}
        aws_secret: ${aws_secret}
        local_file: resmoke-with-binaries.tgz
        # Changed for mongodump_passthrough
        remote_file: mongo-tools/mongodump_passthrough/${build_variant}/${revision}/resmoke-with-binaries/${build_id}.tgz
        bucket: mciuploads
        permissions: private
        visibility: signed
        content_type: application/gzip
        display_name: resmoke-with-binaries

    - command: archive.targz_pack
      params:
        target: resmoke-python-wheelhouse.tgz
        source_dir: "."
        include:
          - "./wheelhouse/**"
          - "./poetry-requirements.txt"
    - command: s3.put
      params:
        aws_key: ${aws_key}
        aws_secret: ${aws_secret}
        local_file: resmoke-python-wheelhouse.tgz
        # Changed for mongodump_passthrough
        remote_file: mongo-tools/mongodump_passthrough/${build_variant}/${revision}/resmoke-python-wheelhouse/${build_id}.tgz
        bucket: mciuploads
        permissions: private
        visibility: signed
        content_type: application/gzip
        display_name: resmoke-python-wheelhouse

  f_migration_verifier_compile_and_upload:
    &f_migration_verifier_compile_and_upload # Compile the migration-verifier:
    - command: shell.exec
      params:
        shell: bash
        working_dir: src/migration-verifier
        script: |
          set -o errexit
          set -o pipefail
          set -o xtrace

          ${PREPARE_SHELL}

          # XXX - Since mise is scoped to the Mongosync repo root, we cannot easily use it to build things
          # outside that root. It'd be nice if the migration-verifier also used mise too, but for now this
          # works okay.
          PATH=$PATH:$HOME:/opt/golang/go1.23/bin
          GOROOT=/opt/golang/go1.23
          ./build.sh
    # upload the compiled verifier to S3.
    - command: s3.put
      params:
        aws_key: ${aws_key}
        aws_secret: ${aws_secret}
        local_file: src/migration-verifier/migration_verifier
        # Changed for mongodump_passthrough
        remote_file: mongo-tools/mongodump_passthrough/${build_variant}/${revision}/${version_id}/migration_verifier
        content_type: application/x-executable
        bucket: mciuploads
        permissions: public-read
        display_name: "migration_verifier release artifact (compiled)"

  f_migration_verifier_binary_fetch:
    - *f_migration_verifier_binary_fetch

  f_make_migration_verifier_binary_executable:
    - *f_make_migration_verifier_binary_executable

  # This uploads everything in src/mongosync/dist/
  # For mongodump, we don't need the mongosync binaries, but we have
  # already built the mongodump and mongorestore binaries and copied
  # them into src/mongosync/dist/
  f_mongosync_binary_upload: &f_mongosync_binary_upload
    - command: archive.targz_pack
      params:
        target: ${mongosync_binary_folder}.tgz
        source_dir: "src/mongosync"
        include:
          - "./dist/*"
    - command: s3.put
      params:
        aws_key: ${aws_key}
        aws_secret: ${aws_secret}
        local_file: ${mongosync_binary_folder}.tgz
        # Changed for mongodump_passthrough
        remote_file: mongo-tools/mongodump_passthrough/${build_variant}/${revision}/${version_id}/${mongosync_binary_folder}/${version_id}.tgz
        bucket: mciuploads
        permissions: public-read
        content_type: application/gzip
        display_name: ${mongosync_binary_folder}

  "passthrough setup":
    - *f_repo_fetch
    # Added for mongodump_passthrough
    - *f_link_to_mongo_tools
    - *f_enable_mongodump_suite_config
    - *f_resmoke_with_binaries_fetch
    - *f_resmoke_wheelhouse_fetch
    - *f_mongosync_binary_fetch
    - *get_buildnumber
    - *f_set_up_logger_credentials

  "run tests":
    - command: expansions.update
      params:
        updates:
          - key: aws_key_remote
            value: ${mongodatafiles_aws_key}
          - key: aws_profile_remote
            value: mongodata_aws
          - key: aws_secret_remote
            value: ${mongodatafiles_aws_secret}
    - *f_expansions_write
    - *f_set_up_archive_credentials
    - *f_resmoke_jobs_determine
    - *f_resmoke_jobs_expansion_update
    - *f_resmoke_tests_execute
    - *f_infrastructure_failure_check

  f_resmoke_report_attach:
    - command: attach.results
      params:
        file_location: src/resmoke/report.json

  f_mongo_coredumps_save:
    - command: subprocess.exec
      params:
        binary: "./src/mongosync/evergreen/scripts/mongo_coredumps_gather.sh"
        add_expansions_to_env: true
    - command: archive.targz_pack
      params:
        target: mongo-coredumps.tgz
        source_dir: *_resmoke_dir
        include:
          - "./**.core"
          - "./**.mdmp" # Windows: minidumps
    - command: s3.put
      params:
        aws_key: ${aws_key}
        aws_secret: ${aws_secret}
        local_file: mongo-coredumps.tgz
        remote_file: mongosync/${build_variant}/${revision}/coredumps/mongo-coredumps-${build_id}-${task_name}-${execution}.tgz
        bucket: mciuploads
        permissions: public-read
        content_type: application/gzip
        display_name: Core Dumps - Execution ${execution}
        optional: true

  "attach artifacts":
    - command: attach.artifacts
      params:
        optional: true
        files:
          - src/resmoke/archive.json
          - src/resmoke/archive-job*.json

    - command: archive.targz_pack
      params:
        target: server-logs.tgz
        source_dir: "src/mongosync/mongosync-testing-server-logs"
        include:
          - "./**"
    - command: s3.put
      params:
        aws_key: ${aws_key}
        aws_secret: ${aws_secret}
        local_file: server-logs.tgz
        remote_file: mongosync/${build_variant}/${revision}/mongo-logs/mongo-logs-${build_id}-${task_name}-${execution}.tgz
        bucket: mciuploads
        permissions: public-read
        content_type: application/gzip
        display_name: Integration Test mongod / mongos logs ${execution}
        optional: true

    - command: archive.targz_pack
      params:
        target: mongo-orchestration-logs.tgz
        source_dir: "src/mongosync"
        include:
          - "out.log"
          - "server.log"
    - command: s3.put
      params:
        aws_key: ${aws_key}
        aws_secret: ${aws_secret}
        local_file: mongo-orchestration-logs.tgz
        remote_file: mongosync/${build_variant}/${revision}/mongo-orchestration-logs/mongo-orchestration-logs.tgz
        bucket: mciuploads
        permissions: public-read
        content_type: application/gzip
        display_name: mongo-orchestration logs
        optional: true

    - command: s3.put
      params:
        aws_key: ${aws_key}
        aws_secret: ${aws_secret}
        local_files_include_filter:
          - src/mongosync/*.suite
        remote_file: mongosync/${build_variant}/${revision}/${build_id}/${task_name}/${execution}/go-test-logs/
        bucket: mciuploads
        permissions: public-read
        content_type: text/plain
        display_name: "Output from `go test` command"

    - command: s3.put
      params:
        aws_key: ${aws_key}
        aws_secret: ${aws_secret}
        local_file: src/mongosync/server-binaries.tgz
        remote_file: mongosync/${build_variant}/${revision}/mongo-binaries/mongo-binaries-${build_id}-${task_name}-${execution}.tgz
        bucket: mciuploads
        permissions: public-read
        content_type: application/gzip
        display_name: Failed Integration Test mongod / mongos binaries ${execution}
        optional: true

    - command: s3.put
      params:
        aws_key: ${aws_key}
        aws_secret: ${aws_secret}
        local_files_include_filter:
          - src/mongosync/mongosync-testing-server-data-files/archive/*.tgz
        remote_file: mongosync/${build_variant}/${revision}/${build_id}/${task_name}/datafiles/
        bucket: mciuploads
        permissions: public-read
        content_type: application/octet-stream
        display_name: Failed Integration Test Data Files ${execution}

    - command: archive.targz_pack
      params:
        target: coverage.gz
        source_dir: "src/mongosync/coverage"
        include:
          - "./**"
    - command: s3.put
      params:
        aws_key: ${aws_key}
        aws_secret: ${aws_secret}
        local_file: coverage.gz
        remote_file: mongosync/${build_variant}/${revision}/coverage/${task_name}/coverage.gz
        bucket: mciuploads
        permissions: public-read
        content_type: application/gzip
        display_name: Coverage statistics

    - command: archive.targz_pack
      params:
        target: tools_output.tgz
        source_dir: "src/resmoke/tools_output"
        include:
          - "**"
    - command: s3.put
      params:
        aws_key: ${aws_key}
        aws_secret: ${aws_secret}
        local_file: tools_output.tgz
        remote_file: mongo-tools/${build_variant}/${revision}/tools_output/${task_name}/${build_id}/tools_output.tgz
        bucket: mciuploads
        permissions: public-read
        content_type: application/gzip
        display_name: Tools Output

  "setup jstestfuzz":
    - *f_expansions_write
    - command: github.generate_token
      params:
        owner: 10gen
        repo: jstestfuzz
        expansion_name: generated_token_jstestfuzz
        permissions:
          contents: read
    - command: subprocess.exec
      params:
        binary: "./src/mongosync/evergreen/scripts/setup_jstestfuzz.sh"
        add_expansions_to_env: true

  "run jstestfuzz":
    - *f_expansions_write
    - *f_generate_github_access_token
    - command: github.generate_token
      params:
        owner: 10gen
        repo: qa
        expansion_name: generated_token_qa
        permissions:
          contents: read
    - command: subprocess.exec
      params:
        binary: "./src/mongosync/evergreen/scripts/clone_jstestfuzz_jstest_repos.sh"
        add_expansions_to_env: true
    - command: subprocess.exec
      type: test
      params:
        binary: "./src/mongosync/evergreen/scripts/run_jstestfuzz.sh"
        add_expansions_to_env: true
    - command: archive.targz_pack
      params:
        target: "jstests.tgz"
        source_dir: "src/jstestfuzz"
        include:
          - "out/*.js"
    - command: s3.put
      params:
        aws_key: ${aws_key}
        aws_secret: ${aws_secret}
        local_file: jstests.tgz
        # Changed for mongodump_passthrough
        remote_file: mongo-tools/mongodump_passthrough/${build_variant}/${revision}/jstestfuzz/${task_id}-${execution}.tgz
        bucket: mciuploads
        permissions: private
        visibility: signed
        content_type: application/gzip
        display_name: JS Fuzz Tests - Execution ${execution}

  "generate mongodump evergreen tasks":
    # Note: This assumes you've already called "fetch source and install go", because you can't call
    # a function from another function.
    - command: shell.exec
      params:
        shell: bash
        working_dir: src/mongosync
        script: |
          set -o errexit
          set -o pipefail
          set -o xtrace

          ${PREPARE_SHELL}
          export aws_key=${aws_key}
          export aws_secret=${aws_secret}
          mise exec -- go run ./cmd/mongodump-task-gen ${generator_args} | tee ${json_filename}
    - command: generate.tasks
      params:
        files:
          - ${json_filename}

  "wait for resmoke to shutdown":
    - command: subprocess.exec
      display_name: "wait for resmoke to shutdown"
      params:
        binary: "./src/mongosync/evergreen/scripts/wait_for_resmoke_to_shutdown.sh"
