diff --git a/.ci/ansible/smash-config.json b/.ci/ansible/smash-config.json index 3e1271fb8d..85dea4fd95 100644 --- a/.ci/ansible/smash-config.json +++ b/.ci/ansible/smash-config.json @@ -6,7 +6,7 @@ ], "selinux enabled": false, "version": "3", - "aiohttp_fixtures_origin": "172.18.0.1" + "aiohttp_fixtures_origin": "127.0.0.1" }, "hosts": [ { @@ -26,7 +26,7 @@ "pulp workers": {}, "redis": {}, "shell": { - "transport": "docker" + "transport": "local" } } } diff --git a/.ci/ansible/start_container.yaml b/.ci/ansible/start_container.yaml index 09da135581..e23a188047 100644 --- a/.ci/ansible/start_container.yaml +++ b/.ci/ansible/start_container.yaml @@ -41,7 +41,7 @@ image: "{{ item.image }}" auto_remove: true recreate: true - privileged: false + privileged: true networks: - name: pulp_ci_bridge aliases: "{{ item.name }}" diff --git a/.ci/assets/bindings/.gitkeep b/.ci/assets/bindings/.gitkeep deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/.github/template_gitref b/.github/template_gitref index aa506a230a..71be7abbf6 100644 --- a/.github/template_gitref +++ b/.github/template_gitref @@ -1 +1 @@ -2021.08.26-139-g0d40f35 +2021.08.26-146-gae643c8 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5730fe74ec..affcc7613b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -133,6 +133,9 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 + - name: Install run: .github/workflows/scripts/install.sh @@ -152,11 +155,6 @@ jobs: run: .github/workflows/scripts/install_python_client.sh shell: bash - - name: Install Ruby client - if: ${{ env.TEST == 'bindings' }} - run: .github/workflows/scripts/install_ruby_client.sh - shell: bash - - name: Before Script run: .github/workflows/scripts/before_script.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 22cba2b5aa..1d9628eb7b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -341,6 +341,7 @@ jobs: set -euv export COMMIT_MSG=$(git log --format=%B --no-merges -1) export GH_ISSUES=$(echo $COMMIT_MSG | grep -o "GH Issues: .*" | awk '{print $3}') + pip install pygithub echo "GH Issues $GH_ISSUES" python .ci/scripts/update_github.py diff --git a/.github/workflows/scripts/install.sh b/.github/workflows/scripts/install.sh index 4176e461bf..fa251fb4bd 100755 --- a/.github/workflows/scripts/install.sh +++ b/.github/workflows/scripts/install.sh @@ -22,9 +22,6 @@ if [[ "$TEST" = "docs" || "$TEST" = "publish" ]]; then pip install -r doc_requirements.txt fi -pip install -e ../pulpcore -e ../pulp_file -e ../pulp-certguard -pip install -r functest_requirements.txt - cd .ci/ansible/ TAG=ci_build @@ -62,6 +59,8 @@ plugins: source: pulp_file - name: pulp-certguard source: pulp-certguard + - name: pulp-smash + source: ./pulp-smash VARSYAML else cat >> vars/main.yaml << VARSYAML @@ -77,6 +76,8 @@ plugins: source: $PULP_CERTGUARD - name: pulpcore source: "${PULPCORE}" + - name: pulp-smash + source: ./pulp-smash VARSYAML fi @@ -87,6 +88,8 @@ services: volumes: - ./settings:/etc/pulp - ./ssh:/keys/ + - ~/.config:/root/.config + - ../../../pulp-openapi-generator:/root/pulp-openapi-generator VARSYAML cat >> vars/main.yaml << VARSYAML diff --git a/.github/workflows/scripts/install_python_client.sh b/.github/workflows/scripts/install_python_client.sh index 6939863cf0..be39f61799 100755 --- a/.github/workflows/scripts/install_python_client.sh +++ b/.github/workflows/scripts/install_python_client.sh @@ -7,7 +7,7 @@ # # For more info visit https://github.com/pulp/plugin_template -set -euv +set -mveuo pipefail export PULP_URL="${PULP_URL:-http://pulp}" @@ -30,7 +30,7 @@ export response=$(curl --write-out %{http_code} --silent --output /dev/null http if [ "$response" == "200" ]; then echo "pulp_rpm client $VERSION has already been released. Installing from PyPI." - pip install pulp-rpm-client==$VERSION + docker exec pulp pip3 install pulp-rpm-client==$VERSION mkdir -p dist tar cvf python-client.tar ./dist exit @@ -41,7 +41,7 @@ rm -rf pulp_rpm-client ./generate.sh pulp_rpm python $VERSION cd pulp_rpm-client python setup.py sdist bdist_wheel --python-tag py3 -find . -name "*.whl" -exec pip install {} \; +find . -name "*.whl" -exec docker exec pulp pip3 install /root/pulp-openapi-generator/pulp_rpm-client/{} \; tar cvf ../../pulp_rpm/python-client.tar ./dist find ./docs/* -exec sed -i 's/Back to README/Back to HOME/g' {} \; diff --git a/.github/workflows/scripts/script.sh b/.github/workflows/scripts/script.sh index e32ea3b85e..badb211193 100755 --- a/.github/workflows/scripts/script.sh +++ b/.github/workflows/scripts/script.sh @@ -61,50 +61,34 @@ if [[ "$TEST" == "plugin-from-pypi" ]]; then git checkout ${COMPONENT_VERSION} -- pulp_rpm/tests/ fi +echo "machine pulp +login admin +password password +" | cmd_stdin_prefix bash -c "cat > /root/.netrc" + +cat unittest_requirements.txt | cmd_stdin_prefix bash -c "cat > /tmp/unittest_requirements.txt" +cat functest_requirements.txt | cmd_stdin_prefix bash -c "cat > /tmp/functest_requirements.txt" +cmd_prefix pip3 install -r /tmp/unittest_requirements.txt +cmd_prefix pip3 install -r /tmp/functest_requirements.txt +cmd_prefix pip3 install --upgrade ../pulp-smash + cd ../pulp-openapi-generator +./generate.sh pulp_rpm python +cmd_prefix pip3 install /root/pulp-openapi-generator/pulp_rpm-client +sudo rm -rf ./pulp_rpm-client ./generate.sh pulpcore python -pip install ./pulpcore-client -rm -rf ./pulpcore-client -if [[ "$TEST" = 'bindings' ]]; then - ./generate.sh pulpcore ruby 0 - cd pulpcore-client - gem build pulpcore_client.gemspec - gem install --both ./pulpcore_client-0.gem -fi +cmd_prefix pip3 install /root/pulp-openapi-generator/pulpcore-client +sudo rm -rf ./pulpcore-client ./generate.sh pulp_file python -pip install ./pulp_file-client -rm -rf ./pulp_file-client -if [[ "$TEST" = 'bindings' ]]; then - ./generate.sh pulp_file ruby 0 - cd pulp_file-client - gem build pulp_file_client.gemspec - gem install --both ./pulp_file_client-0.gem - cd .. -fi +cmd_prefix pip3 install /root/pulp-openapi-generator/pulp_file-client +sudo rm -rf ./pulp_file-client ./generate.sh pulp_certguard python -pip install ./pulp_certguard-client -rm -rf ./pulp_certguard-client -if [[ "$TEST" = 'bindings' ]]; then - ./generate.sh pulp-certguard ruby 0 - cd pulp-certguard-client - gem build pulp-certguard_client.gemspec - gem install --both ./pulp-certguard_client-0.gem - cd .. -fi +cmd_prefix pip3 install /root/pulp-openapi-generator/pulp_certguard-client +sudo rm -rf ./pulp_certguard-client cd $REPO_ROOT -if [[ "$TEST" = 'bindings' ]]; then - if [ -f $REPO_ROOT/.ci/assets/bindings/test_bindings.py ]; then - python $REPO_ROOT/.ci/assets/bindings/test_bindings.py - fi - if [ -f $REPO_ROOT/.ci/assets/bindings/test_bindings.rb ]; then - ruby $REPO_ROOT/.ci/assets/bindings/test_bindings.rb - fi - exit -fi - -cat unittest_requirements.txt | cmd_stdin_prefix bash -c "cat > /tmp/unittest_requirements.txt" -cmd_prefix pip3 install -r /tmp/unittest_requirements.txt +CERTIFI=$(cmd_prefix python3 -c 'import certifi; print(certifi.where())') +cmd_prefix bash -c "cat /etc/pulp/certs/pulp_webserver.crt | tee -a "$CERTIFI" > /dev/null" # check for any uncommitted migrations echo "Checking for uncommitted migrations..." @@ -116,15 +100,11 @@ if [[ "$TEST" != "upgrade" ]]; then fi # Run functional tests -export PYTHONPATH=$REPO_ROOT/../pulp_file${PYTHONPATH:+:${PYTHONPATH}} -export PYTHONPATH=$REPO_ROOT/../pulp-certguard${PYTHONPATH:+:${PYTHONPATH}} -export PYTHONPATH=$REPO_ROOT${PYTHONPATH:+:${PYTHONPATH}} - if [[ "$TEST" == "performance" ]]; then if [[ -z ${PERFORMANCE_TEST+x} ]]; then - pytest -vv -r sx --color=yes --pyargs --capture=no --durations=0 pulp_rpm.tests.performance + cmd_prefix bash -c "pytest -vv -r sx --color=yes --pyargs --capture=no --durations=0 pulp_rpm.tests.performance" else - pytest -vv -r sx --color=yes --pyargs --capture=no --durations=0 pulp_rpm.tests.performance.test_$PERFORMANCE_TEST + cmd_prefix bash -c "pytest -vv -r sx --color=yes --pyargs --capture=no --durations=0 pulp_rpm.tests.performance.test_$PERFORMANCE_TEST" fi exit fi @@ -133,21 +113,21 @@ if [ -f $FUNC_TEST_SCRIPT ]; then source $FUNC_TEST_SCRIPT else - if [[ "$GITHUB_WORKFLOW" == "Rpm Nightly CI/CD" ]]; then - pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_rpm.tests.functional -m parallel -n 8 --nightly - pytest -v -r sx --color=yes --pyargs pulp_rpm.tests.functional -m "not parallel" --nightly + if [[ "$GITHUB_WORKFLOW" == "Rpm Nightly CI/CD" ]] || [[ "${RELEASE_WORKFLOW:-false}" == "true" ]]; then + cmd_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_rpm.tests.functional -m parallel -n 8 --nightly" + cmd_prefix bash -c "pytest -v -r sx --color=yes --pyargs pulp_rpm.tests.functional -m 'not parallel' --nightly" - pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulpcore.tests.functional -m "from_pulpcore_for_all_plugins and parallel" -n 8 --nightly - pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulpcore.tests.functional -m "from_pulpcore_for_all_plugins and not parallel" --nightly + cmd_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulpcore.tests.functional -m 'from_pulpcore_for_all_plugins and parallel' -n 8 --nightly" + cmd_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulpcore.tests.functional -m 'from_pulpcore_for_all_plugins and not parallel' --nightly" else - pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_rpm.tests.functional -m parallel -n 8 - pytest -v -r sx --color=yes --pyargs pulp_rpm.tests.functional -m "not parallel" + cmd_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_rpm.tests.functional -m parallel -n 8" + cmd_prefix bash -c "pytest -v -r sx --color=yes --pyargs pulp_rpm.tests.functional -m 'not parallel'" - pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulpcore.tests.functional -m "from_pulpcore_for_all_plugins and parallel" -n 8 - pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulpcore.tests.functional -m "from_pulpcore_for_all_plugins and not parallel" + cmd_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulpcore.tests.functional -m 'from_pulpcore_for_all_plugins and parallel' -n 8" + cmd_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulpcore.tests.functional -m 'from_pulpcore_for_all_plugins and not parallel'" fi diff --git a/requirements.txt b/requirements.txt index a74597846b..dfd79c7974 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ createrepo_c~=0.20.0 django_readonly_field~=1.1.1 -jsonschema~=4.6 +jsonschema~=4.6.0 libcomps~=0.1.15 productmd~=1.33.0 pulpcore~=3.14.5