Exporting environment variables Exporting CI Exporting DISPLAY Exporting PAGER Exporting SEMAPHORE Exporting SEMAPHORE_AGENT_MACHINE_ENVIRONMENT_TYPE Exporting SEMAPHORE_AGENT_MACHINE_OS_IMAGE Exporting SEMAPHORE_AGENT_MACHINE_TYPE Exporting SEMAPHORE_ARTIFACT_TOKEN Exporting SEMAPHORE_CACHE_ARCHIVE_METHOD Exporting SEMAPHORE_CACHE_BACKEND Exporting SEMAPHORE_CACHE_PRIVATE_KEY_PATH Exporting SEMAPHORE_CACHE_URL Exporting SEMAPHORE_CACHE_USERNAME Exporting SEMAPHORE_GIT_BRANCH Exporting SEMAPHORE_GIT_COMMIT_RANGE Exporting SEMAPHORE_GIT_DIR Exporting SEMAPHORE_GIT_PROVIDER Exporting SEMAPHORE_GIT_REF Exporting SEMAPHORE_GIT_REF_TYPE Exporting SEMAPHORE_GIT_REPO_NAME Exporting SEMAPHORE_GIT_REPO_SLUG Exporting SEMAPHORE_GIT_SHA Exporting SEMAPHORE_GIT_URL Exporting SEMAPHORE_GIT_WORKING_BRANCH Exporting SEMAPHORE_JOB_ID Exporting SEMAPHORE_JOB_NAME Exporting SEMAPHORE_ORGANIZATION_URL Exporting SEMAPHORE_PIPELINE_0_ARTEFACT_ID Exporting SEMAPHORE_PIPELINE_ARTEFACT_ID Exporting SEMAPHORE_PIPELINE_ID Exporting SEMAPHORE_PIPELINE_PROMOTED_BY Exporting SEMAPHORE_PIPELINE_PROMOTION Exporting SEMAPHORE_PIPELINE_RERUN Exporting SEMAPHORE_PROJECT_ID Exporting SEMAPHORE_PROJECT_NAME Exporting SEMAPHORE_TOOLBOX_METRICS_ENABLED Exporting SEMAPHORE_WORKFLOW_HOOK_SOURCE Exporting SEMAPHORE_WORKFLOW_ID Exporting SEMAPHORE_WORKFLOW_NUMBER Exporting SEMAPHORE_WORKFLOW_RERUN Exporting SEMAPHORE_WORKFLOW_TRIGGERED_BY_API Exporting SEMAPHORE_WORKFLOW_TRIGGERED_BY_HOOK Exporting SEMAPHORE_WORKFLOW_TRIGGERED_BY_SCHEDULE Exporting SEMAPHORE_YAML_FILE_PATH Exporting SSH_PRIVATE_KEY_PATH Exporting TERM Injecting Files Injecting /home/semaphore/.ssh/semaphore_cache_key with file mode 0600 Injecting /home/semaphore/.ssh/id_rsa with file mode 0600 Setting up the Semaphore toolbox Running: dowloadPath=$(curl -s -L --retry 5 http://packages.semaphoreci.com/toolbox/linux) && echo $dowloadPath && curl -s -L --retry 5 --connect-timeout 3 $dowloadPath -o /tmp/toolbox.tar && tar -xvf /tmp/toolbox.tar && mv toolbox ~/.toolbox && bash ~/.toolbox/install-toolbox && source ~/.toolbox/toolbox && echo 'source ~/.toolbox/toolbox' >> ~/.bash_profile && if [ -f /etc/init.d/xvfb ]; then /etc/init.d/xvfb start; fi http://packages.semaphoreci.com/toolbox/releases/download/v1.19.25/linux.tar toolbox/ toolbox/enetwork toolbox/sem-service-check-params toolbox/sem-version toolbox/sem-dockerize toolbox/artifact toolbox/sem-context toolbox/test-results toolbox/sem-semantic-release toolbox/retry toolbox/install-toolbox toolbox/cache toolbox/install-package toolbox/system-metrics-collector toolbox/libchecksum toolbox/README.md toolbox/when toolbox/toolbox toolbox/spc toolbox/sem-install toolbox/ssh-session-cli toolbox/libcheckout toolbox/sem-service toolbox/Dockerfile.dev retry installed ssh-session-cli installed cache installed sem-context installed sem-service installed sem-dockerize installed sem-service-check-params installed sem-semantic-release installed install-package installed Installing the artifacts CLI artifacts installed Installing the test results CLI test-results installed Installing the SPC CLI spc installed Installing the when CLI when installed Installing the enetwork CLI when installed Starting to collect System Metrics in /tmp/system-metrics Starting virtual X frame buffer: Xvfb. Starting an ssh-agent Running: eval `ssh-agent` && echo 'eval $(ssh-agent) >/dev/null' >> ~/.bash_profile && echo 'yes' > /tmp/sempahore-user-commands-have-started Agent pid 3555 Connecting to cache Running: (ssh-keyscan -p 29920 -H 116.202.32.43 >> /home/semaphore/.ssh/known_hosts || true) && ssh-add /home/semaphore/.ssh/semaphore_cache_key # 116.202.32.43:29920 SSH-2.0-OpenSSH_7.6p1 Ubuntu-4ubuntu0.7 # 116.202.32.43:29920 SSH-2.0-OpenSSH_7.6p1 Ubuntu-4ubuntu0.7 # 116.202.32.43:29920 SSH-2.0-OpenSSH_7.6p1 Ubuntu-4ubuntu0.7 Identity added: /home/semaphore/.ssh/semaphore_cache_key (/home/semaphore/.ssh/semaphore_cache_key) export GIT_LFS_SKIP_SMUDGE=1 checkout Performing shallow clone with depth: 50 HEAD is now at d2862c0 updated submodule bash-tools export INPUT_FILE="$SEMAPHORE_YAML_FILE_PATH" export OUTPUT_FILE="${SEMAPHORE_YAML_FILE_PATH}.output.yml" export LOGS_FILE="${SEMAPHORE_YAML_FILE_PATH}.logs.jsonl" cat $INPUT_FILE # # Author: Hari Sekhon # Date: 2020-03-16 14:02:53 +0000 (Mon, 16 Mar 2020) # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/pylib # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # https://docs.semaphoreci.com/reference/pipeline-yaml-reference/ version: v1.0 name: pylib agent: machine: type: e1-standard-2 os_image: ubuntu1804 execution_time_limit: hours: 3 blocks: - name: Linux build run: when: "branch = 'master' AND change_in('/', {exclude: ['**/*.md']})" #execution_time_limit: # hours: 2 task: #env_vars: # $PATH selects /usr/bin/python and /usr/local/bin/pip which are mismatched versions of Python #- name: PYTHON # value: python3 #- name: PIP # value: pip3 prologue: commands: - cache restore # each job is separate and could be run on a separate machine so all steps must be together jobs: - name: build commands: - checkout - setup/ci_bootstrap.sh - make init - make ci - make test epilogue: commands: - cache store - name: Mac build run: when: "branch = 'master'" task: # because otherwise on Mac it uses /usr/bin/python (2.7) but /usr/local/bin/pip (python 3.8) #env_vars: # to match /usr/local/bin/pip version from $PATH #- name: PYTHON # value: python3 # must be quoted to force string, otherwise pipeline fails to run with this parsing error: # Error: [{"Type mismatch. Expected String but got Integer.", "#/blocks/1/task/env_vars/1/value"}] #- name: DEBUG # value: "1" agent: machine: type: a1-standard-4 os_image: macos-xcode12 prologue: commands: - cache restore # fix for: # pip is configured with locations that require TLS/SSL, however the ssl module in Python is not available. - brew install openssl - brew reinstall python - brew reinstall wget # avoid Mac SSL errors: # # ERROR: Loading command: install (LoadError) # dlopen(/Users/semaphore/.rbenv/versions/2.5.1/lib/ruby/2.5.0/x86_64-darwin18/openssl.bundle, 9): Library not loaded: /usr/local/opt/openssl/lib/libssl.1.0.0.dylib # Referenced from: /Users/semaphore/.rbenv/versions/2.5.1/lib/ruby/2.5.0/x86_64-darwin18/openssl.bundle # Reason: image not found - /Users/semaphore/.rbenv/versions/2.5.1/lib/ruby/2.5.0/x86_64-darwin18/openssl.bundle # ERROR: While executing gem ... (NoMethodError) # undefined method `invoke_with_build_args' for nil:NilClass# # - rbenv global system # also considered this: # - for version in $(rbenv versions | grep -v system | sed 's/^\*//'); do yes | rbenv uninstall "$version"; rbenv install "$version"; done # # fix for python vs pip version mismatch - ln -svf -- /usr/local/bin/python3 /usr/local/bin/python jobs: - name: build commands: - checkout - make init - make ci - make test epilogue: commands: - cache store echo "Compiling $INPUT_FILE into $OUTPUT_FILE and storring logs to $LOGS_FILE" Compiling .semaphore/semaphore.yml into .semaphore/semaphore.yml.output.yml and storring logs to .semaphore/semaphore.yml.logs.jsonl spc compile --input $INPUT_FILE --output $OUTPUT_FILE --logs $LOGS_FILE Evaluating parameters expressions in .semaphore/semaphore.yml. Found parameters expressions at 0 locations. Substituting parameters with their values. Evaluating change_in expressions in .semaphore/semaphore.yml. Found when expressions at 2 locations. 001 | Location: [blocks 0 run when] | File: .semaphore/semaphore.yml | Expression: branch = 'master' AND change_in('/', {exclude: ['**/*.md']}) 002 | Location: [blocks 1 run when] | File: .semaphore/semaphore.yml | Expression: branch = 'master' Evaluating when expressions. 001 | When Expression: branch = 'master' AND change_in('/', {exclude: ['**/*.md']}) | | change_in(["/",{"exclude":["**/*.md"]}]) | Running git fetch origin +refs/heads/master:refs/heads/master | Running git diff --shortstat master...d2862c007f4bf32388ca16f682991425425e26fc | Running git diff --name-only master...d2862c007f4bf32388ca16f682991425425e26fc | | Comparing change_in with git diff | (match) | | Result: true 002 | When Expression: branch = 'master' Exporting environment variables Exporting SEMAPHORE_JOB_RESULT export BASE_NAME=$SEMAPHORE_PIPELINE_ID-$(basename $INPUT_FILE) export ARTIFACT_LOG_DESTINATION="compilation/$BASE_NAME.logs" echo "Uploading $LOGS_FILE into $ARTIFACT_LOG_DESTINATION" Uploading .semaphore/semaphore.yml.logs.jsonl into compilation/3609a11b-73e5-4f0b-b213-726132c4a9e1-semaphore.yml.logs artifact push workflow $LOGS_FILE -d $ARTIFACT_LOG_DESTINATION 2023/02/14 07:43:08 [DEBUG] POST https://harisekhon.semaphoreci.com/api/v1/artifacts [Feb 14 07:43:08.560] Successfully pushed artifact for current workflow. [Feb 14 07:43:08.560] * Local source: .semaphore/semaphore.yml.logs.jsonl. [Feb 14 07:43:08.561] * Remote destination: artifacts/workflows/3826eefc-675a-45c7-a62d-8d7a323564e7/compilation/3609a11b-73e5-4f0b-b213-726132c4a9e1-semaphore.yml.logs. export ARTIFACT_YAML_DESTINATION="compilation/$BASE_NAME" echo "Uploading $OUTPUT_FILE into $ARTIFACT_YAML_DESTINATION" Uploading .semaphore/semaphore.yml.output.yml into compilation/3609a11b-73e5-4f0b-b213-726132c4a9e1-semaphore.yml artifact push workflow $OUTPUT_FILE -d $ARTIFACT_YAML_DESTINATION 2023/02/14 07:43:08 [DEBUG] POST https://harisekhon.semaphoreci.com/api/v1/artifacts [Feb 14 07:43:09.233] Successfully pushed artifact for current workflow. [Feb 14 07:43:09.233] * Local source: .semaphore/semaphore.yml.output.yml. [Feb 14 07:43:09.233] * Remote destination: artifacts/workflows/3826eefc-675a-45c7-a62d-8d7a323564e7/compilation/3609a11b-73e5-4f0b-b213-726132c4a9e1-semaphore.yml.