tahoe-lafs/.circleci/config.yml

# https://circleci.com/docs/2.0/

version: 2
workflows:
  version: 2
  ci:
    jobs:
      # Platforms
      - "debian-9"
      - "debian-8":
          requires:
            - "debian-9"

      - "ubuntu-18.04"
      - "ubuntu-16.04":
          requires:
            - "ubuntu-18.04"

      - "fedora-29"
      - "fedora-28":
          requires:
            - "fedora-29"

      - "centos-7"

      - "slackware-14.2"

      # Other assorted tasks and configurations
      - "lint"
      - "deprecations"
      - "c-locale"
      # Any locale other than C or UTF-8.
      - "another-locale"

      - "integration":
          requires:
            # If the unit test suite doesn't pass, don't bother running the
            # integration tests.
            - "debian-9"


jobs:
  lint:
    docker:
      - image: "circleci/python:2"

    steps:
      - "checkout"

      - run:
          name: "Install tox"
          command: |
            pip install --user tox

      - run:
          name: "Static-ish code checks"
          command: |
            ~/.local/bin/tox -e codechecks

  debian-9: &DEBIAN
    docker:
      - image: "tahoelafsci/debian:9"

    environment: &UTF_8_ENVIRONMENT
      # Tell Hypothesis which configuration we want it to use.
      TAHOE_LAFS_HYPOTHESIS_PROFILE: "ci"
      # Tell the C runtime things about character encoding (mainly to do with
      # filenames and argv).
      LANG: "en_US.UTF-8"
      # Select a tox environment to run for this job.
      TAHOE_LAFS_TOX_ENVIRONMENT: "coverage"
      # Additional arguments to pass to tox.
      TAHOE_LAFS_TOX_ARGS: ""
      # The path in which test artifacts will be placed.
      ARTIFACTS_OUTPUT_PATH: "/tmp/artifacts"
      # Convince all of our pip invocations to look at the cached wheelhouse
      # we maintain.
      WHEELHOUSE_PATH: &WHEELHOUSE_PATH "/tmp/wheelhouse"
      PIP_FIND_LINKS: "file:///tmp/wheelhouse"

    # pip cannot install packages if the working directory is not readable.
    # We want to run a lot of steps as nobody instead of as root.
    working_directory: "/tmp/project"

    steps:
      - "checkout"
      - run:
          name: "Fix permissions"
          command: |
            chown --recursive nobody:nogroup /tmp/project
      - run: &SETUP_VIRTUALENV
          name: "Setup virtualenv"
          command: |
            sudo --set-home -u nobody /tmp/project/.circleci/setup-virtualenv.sh \
                "/tmp/venv" \
                "/tmp/project" \
                "${WHEELHOUSE_PATH}" \
                "${TAHOE_LAFS_TOX_ENVIRONMENT}" \
                "${TAHOE_LAFS_TOX_ARGS}"

      - run: &RUN_TESTS
          name: "Run test suite"
          command: |
            /tmp/project/.circleci/run-tests.sh \
                "/tmp/venv" \
                "/tmp/project" \
                "${ARTIFACTS_OUTPUT_PATH}" \
                "${TAHOE_LAFS_TOX_ENVIRONMENT}" \
                "${TAHOE_LAFS_TOX_ARGS}"
          # trial output gets directed straight to a log.  avoid the circleci
          # timeout while the test suite runs.
          no_output_timeout: "20m"

      - store_test_results: &STORE_TEST_RESULTS
          path: "/tmp/artifacts/junit"

      - store_artifacts: &STORE_TEST_LOG
          # Despite passing --workdir /tmp to tox above, it still runs trial
          # in the project source checkout.
          path: "/tmp/project/_trial_temp/test.log"

      - store_artifacts: &STORE_OTHER_ARTIFACTS
          # Store any other artifacts, too.  This is handy to allow other jobs
          # sharing most of the definition of this one to be able to
          # contribute artifacts easily.
          path: "/tmp/artifacts"

      - run: &SUBMIT_COVERAGE
          name: "Submit coverage results"
          command: |
            /tmp/venv/bin/codecov


  debian-8:
    <<: *DEBIAN
    docker:
      - image: "tahoelafsci/debian:8"


  c-locale:
    <<: *DEBIAN

    environment:
      <<: *UTF_8_ENVIRONMENT
      LANG: "C"


  another-locale:
    <<: *DEBIAN

    environment:
      <<: *UTF_8_ENVIRONMENT
      # aka "Latin 1"
      LANG: "en_US.ISO-8859-1"


  deprecations:
    <<: *DEBIAN

    environment:
      <<: *UTF_8_ENVIRONMENT
      # Select the deprecations tox environments.
      TAHOE_LAFS_TOX_ENVIRONMENT: "deprecations,upcoming-deprecations"
      # Put the logs somewhere we can report them.
      TAHOE_LAFS_WARNINGS_LOG: "/tmp/artifacts/deprecation-warnings.log"


  integration:
    <<: *DEBIAN

    environment:
      <<: *UTF_8_ENVIRONMENT
      # Select the integration tests tox environments.
      TAHOE_LAFS_TOX_ENVIRONMENT: "integration"
      # Disable artifact collection because py.test can't produce any.
      ARTIFACTS_OUTPUT_PATH: ""


    steps:
      - "checkout"
      - run:
          name: "Bootstrap test environment"
          command: |
            ~/project/.circleci/bootstrap-test-environment.sh ~/project
      # DRY, YAML-style.  See the debian-9 steps.
      - run: *SETUP_VIRTUALENV
      - run:
          name: "Install Tor"
          command: |
            # Don't forget that we moved the source tree to /tmp.
            /tmp/project/integration/install-tor.sh
      - run: *RUN_TESTS


  ubuntu-16.04:
    <<: *DEBIAN
    docker:
      - image: "tahoelafsci/ubuntu:16.04"


  ubuntu-18.04:
    <<: *DEBIAN
    docker:
      - image: "tahoelafsci/ubuntu:18.04"


  centos-7: &RHEL_DERIV
    docker:
      - image: "centos:7"

    environment: *UTF_8_ENVIRONMENT

    steps:
      - run:
          name: "Install Git"
          command: |
            yum install --assumeyes git

      - "checkout"

      - run:
          name: "Bootstrap test environment"
          working_directory: "/tmp"
          command: |
            # Avoid the /nonexistent home directory in nobody's /etc/passwd
            # entry.
            usermod --home /tmp/nobody nobody

            # Grant read access to nobody, the user which will eventually try
            # to test this checkout.
            mv /root/project /tmp/project

            # Python build/install toolchain wants to write to the source
            # checkout, too.
            chown --recursive nobody:nobody /tmp/project

            yum install --assumeyes \
                sudo \
                make automake gcc gcc-c++ \
                python \
                python-devel \
                libffi-devel \
                openssl-devel \
                libyaml-devel
            yum install --assumeyes /usr/bin/virtualenv

            # XXX net-tools is actually a Tahoe-LAFS runtime dependency!
            yum install --assumeyes \
                net-tools

      - run: *SETUP_VIRTUALENV

      - run: *RUN_TESTS

      - store_test_results: *STORE_TEST_RESULTS
      - store_artifacts: *STORE_TEST_LOG
      - store_artifacts: *STORE_OTHER_ARTIFACTS
      - run: *SUBMIT_COVERAGE


  fedora-28:
    <<: *RHEL_DERIV
    docker:
      - image: "fedora:28"


  fedora-29:
    <<: *RHEL_DERIV
    docker:
      - image: "fedora:29"


  slackware-14.2:
    docker:
      - image: "vbatts/slackware:14.2"

    environment: *UTF_8_ENVIRONMENT

    steps:
      - run:
          name: "Install Git"
          command: |
            slackpkg update

            # Be careful with slackpkg.  If the package name given doesn't
            # match anything, slackpkg still claims to succeed but you're
            # totally screwed.  Slackware updates versions of packaged
            # software so including too much version prefix is a good way to
            # have your install commands suddenly begin not installing
            # anything.
            slackpkg install openssh-7 git-2 </dev/null

      - "checkout"

      - run:
          name: "Bootstrap test environment"
          working_directory: "/tmp"
          command: |
            # Avoid the /nonexistent home directory in nobody's /etc/passwd
            # entry.
            usermod --home /tmp/nobody nobody

            # Grant read access to nobody, the user which will eventually try
            # to test this checkout.
            mv /root/project /tmp/project

            # Python build/install toolchain wants to write to the source
            # checkout, too.
            chown --recursive nobody:nobody /tmp/project

            slackpkg install \
                ca-certificates \
                sudo-1 \
                make-4 \
                automake-1 \
                kernel-headers \
                glibc-2 \
                binutils-2 \
                gcc-5 \
                gcc-g++-5 \
                python-2 \
                libffi-3 \
                libyaml-0 \
                sqlite-3 \
                icu4c-56 \
                libmpc-1 </dev/null

            slackpkg upgrade \
                openssl-1 </dev/null

            # neither virtualenv nor pip is packaged.
            # do it the hard way.
            # and it is extra hard since it is slackware.
            slackpkg install \
                cyrus-sasl-2 \
                curl-7 </dev/null
            curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
            python get-pip.py
            pip install virtualenv

      - run: *SETUP_VIRTUALENV

      - run: *RUN_TESTS

      - store_test_results: *STORE_TEST_RESULTS
      - store_artifacts: *STORE_TEST_LOG
      - store_artifacts: *STORE_OTHER_ARTIFACTS
      - run: *SUBMIT_COVERAGE

  build-images:
    # This job builds Docker images that have as much of the setup as we can
    # manage already done and baked in.  This cuts down on the per-job setup
    # time the actual testing jobs have to perform - by perhaps 10% - 20%.
    #
    # https://circleci.com/blog/how-to-build-a-docker-image-on-circleci-2-0/
    docker:
      - image: "docker:17.05.0-ce-git"

    steps:
      - "checkout"
      - "setup_remote_docker"
      - run:
          name: "Get openssl"
          command: |
            apk add --no-cache openssl
      - run:
          name: "Get Dockerhub secrets"
          command: |
            # If you create an encryption key like this:
            #
            #   openssl enc -aes-256-cbc -k secret -P -md sha256

            # From the output that looks like:
            #
            #  salt=...
            #  key=...
            #  iv =...
            #
            # extract just the value for ``key``.

            # then you can re-generate ``secret-env-cipher`` locally using the
            # command:
            #
            #   openssl aes-256-cbc -e -md sha256 -in secret-env-plain -out .circleci/secret-env-cipher -pass env:KEY
            #
            # Make sure the key is set as the KEY environment variable in the
            # CircleCI web interface.  You can do this by visiting
            # <https://circleci.com/gh/tahoe-lafs/tahoe-lafs/edit#env-vars>
            # after logging in to CircleCI with an account in the tahoe-lafs
            # CircleCI team.
            #
            # Then you can recover the environment plaintext (for example, to
            # change and re-encrypt it) like just like CircleCI recovers it
            # here:
            #
            openssl aes-256-cbc -d -md sha256 -in .circleci/secret-env-cipher -pass env:KEY >> ~/.env
      - run:
          name: "Log in to Dockerhub"
          command: |
            . ~/.env
            # TAHOELAFSCI_PASSWORD come from the secret env.
            docker login -u tahoelafsci -p ${TAHOELAFSCI_PASSWORD}
      - run:
          name: "Build Debian images"
          command: |
            docker build -t tahoelafsci/debian:8 -f ~/project/.circleci/Dockerfile.debian-8 ~/project/
            docker build -t tahoelafsci/debian:9 -f ~/project/.circleci/Dockerfile.debian-9 ~/project/
      - run:
          name: "Push Debian images"
          command: |
            # You have to create the debian repository (presumably via the
            # Dockerhub web interface) before anything can be pushed to it.
            docker push tahoelafsci/debian:8
            docker push tahoelafsci/debian:9
      - run:
          name: "Build Ubuntu images"
          command: |
            docker build -t tahoelafsci/ubuntu:16.04 -f ~/project/.circleci/Dockerfile.ubuntu-16.04 ~/project/
            docker build -t tahoelafsci/ubuntu:18.04 -f ~/project/.circleci/Dockerfile.ubuntu-18.04 ~/project/
      - run:
          name: "Push Ubuntu images"
          command: |
            # You have to create the ubuntu repository (presumably via the
            # Dockerhub web interface) before anything can be pushed to it.
            docker push tahoelafsci/ubuntu:16.04
            docker push tahoelafsci/ubuntu:18.04