scrapegraph-py/.github/workflows/test.yml at main · ScrapeGraphAI/scrapegraph-py · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
name: Test Python SDK

on:
  push:
    branches: [ main, master ]
  pull_request:
    branches: [ main, master ]

jobs:
  test:
    runs-on: ubuntu-latest

    steps:
    - uses: actions/checkout@v4

    - name: Set up Python
      uses: actions/setup-python@v4
      with:
        python-version: "3.12"

    - name: Cache pip dependencies
      uses: actions/cache@v3
      with:
        path: ~/.cache/pip
        key: ${{ runner.os }}-pip-3.12-${{ hashFiles('**/pyproject.toml') }}
        restore-keys: |
          ${{ runner.os }}-pip-3.12-

    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install pytest pytest-asyncio responses aioresponses
        cd scrapegraph-py
        pip install -e ".[html]"

    - name: Run tests
      run: |
        cd scrapegraph-py
        pytest tests/ -v --ignore=tests/test_integration_v2.py
    - name: Upload coverage to Codecov
      uses: codecov/codecov-action@v3
      with:
        file: ./scrapegraph-py/coverage.xml
        flags: unittests
        name: codecov-umbrella
        fail_ci_if_error: false

  lint:
    runs-on: ubuntu-latest

    steps:
    - uses: actions/checkout@v4

    - name: Set up Python
      uses: actions/setup-python@v4
      with:
        python-version: "3.11"

    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install flake8 black isort mypy
        cd scrapegraph-py
        pip install -e .

    - name: Run linting
      run: |
        cd scrapegraph-py
        flake8 scrapegraph_py/ tests/ --max-line-length=120 --extend-ignore=E203,W503,E501,F401,F841
        black --check scrapegraph_py/ tests/
        isort --check-only scrapegraph_py/ tests/
        mypy scrapegraph_py/ --ignore-missing-imports

  security:
    runs-on: ubuntu-latest

    steps:
    - uses: actions/checkout@v4

    - name: Set up Python
      uses: actions/setup-python@v4
      with:
        python-version: "3.11"

    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install bandit safety
        cd scrapegraph-py
        pip install -e .

    - name: Run security checks
      run: |
        cd scrapegraph-py
        bandit -r scrapegraph_py/ -f json -o bandit-report.json || true
        safety check --json --output safety-report.json || true