Version upgrade

.devcontainer/devcontainer.json

@@ -24,11 +24,11 @@
     "ghcr.io/jungaretti/features/make:1": {},
     "ghcr.io/devcontainers/features/docker-in-docker:2.12.0": {},
     "ghcr.io/devcontainers/features/python:1": {
-      "version": "3.9"
+      "version": "3.11"
     },
     "ghcr.io/devcontainers-extra/features/poetry:2": {},
     "ghcr.io/devcontainers/features/node:1": {
-      "version": "18"
+      "version": "24"
     }
   },
   "forwardPorts": [

.devcontainer/docker-compose.yml

@@ -24,7 +24,7 @@ services:
       - default
     command: sleep infinity
   mysql:
-    image: mysql:8
+    image: mysql:8.4.10
     volumes:
       - mysql-storage:/var/lib/mysql
     restart: always
@@ -42,12 +42,18 @@ services:
   grafana:
     image: devlake.docker.scarf.sh/apache/devlake-dashboard:latest
     ports:
-      - 3002:3000
+      - 3001:3000
     volumes:
       - grafana-storage:/var/lib/grafana
     environment:
       GF_SERVER_ROOT_URL: "http://localhost:4000/grafana"
       GF_USERS_DEFAULT_THEME: "light"
+      DATABASE_TYPE: mysql
+      DATABASE_HOST: mysql
+      DATABASE_PORT: 3306
+      DATABASE_NAME: lake
+      DATABASE_USER: merico
+      DATABASE_PASSWORD: merico
       MYSQL_URL: mysql:3306
       MYSQL_DATABASE: lake
       MYSQL_USER: merico

.github/workflows/build.yml

@@ -19,6 +19,9 @@ name: Build-Images-Push-Docker
 env:
   DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USER }}
   DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
+  DEVLAKE_BACKEND_DOCKERFILE: ./backend/Dockerfile
+  DEVLAKE_CONFIG_UI_DOCKERFILE: ./config-ui/Dockerfile
+  DEVLAKE_GRAFANA_DOCKERFILE: ./grafana/Dockerfile
 on:
   push:
     tags:
@@ -55,6 +58,7 @@ jobs:
         uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f  # v7.1.0
         with:
           context: ./backend
+          file: ${{ env.DEVLAKE_BACKEND_DOCKERFILE }}
           push: true
           target: builder
           tags: ${{ secrets.DOCKERHUB_OWNER }}/devlake:amd64-builder
@@ -87,6 +91,7 @@ jobs:
         uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f  # v7.1.0
         with:
           context: ./backend
+          file: ${{ env.DEVLAKE_BACKEND_DOCKERFILE }}
           push: true
           target: base
           tags: ${{ secrets.DOCKERHUB_OWNER }}/devlake:base
@@ -130,6 +135,7 @@ jobs:
         uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f  # v7.1.0
         with:
           context: ./backend
+          file: ${{ env.DEVLAKE_BACKEND_DOCKERFILE }}
           push: false
           target: build
           tags: ${{ secrets.DOCKERHUB_OWNER }}/devlake:build-cache-${{ matrix.platform }}
@@ -197,6 +203,7 @@ jobs:
         uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f  # v7.1.0
         with:
           context: ./backend
+          file: ${{ env.DEVLAKE_BACKEND_DOCKERFILE }}
           push: true
           tags: ${{ steps.get_push_tags.outputs.TAGS }}
           platforms: linux/amd64,linux/arm64
@@ -236,9 +243,11 @@ jobs:
           - name: config-ui
             image: devlake-config-ui
             context: config-ui
+            dockerfile: ./config-ui/Dockerfile
           - name: grafana
             image: devlake-dashboard
             context: grafana
+            dockerfile: ./grafana/Dockerfile
     steps:
       - name: Free Disk Space
         run: |
@@ -276,6 +285,7 @@ jobs:
         uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f  # v7.1.0
         with:
           context: ${{ matrix.build.context }}
+          file: ${{ matrix.build.dockerfile }}
           push: true
           tags: ${{ steps.get_push_tags.outputs.TAGS }}
           platforms: linux/amd64,linux/arm64

.github/workflows/config-ui.yml

@@ -29,7 +29,7 @@ jobs:
     - name: Set up Node.js
       uses: actions/setup-node@v6
       with:
-        node-version: '18'
+        node-version: '24'
         cache: 'yarn'
         cache-dependency-path: config-ui/yarn.lock
     - name: Install with Yarn

.gitignore

@@ -168,4 +168,6 @@ venv
 /backend/plugins/gitextractor/parser/output*
 /backend/plugins/gitextractor/parser/demo_repo/
 
-__debug_bin*
+__debug_bin*
+/scripts-local/
+

AGENTS.md

@@ -52,6 +52,25 @@ See [backend/plugins/gitlab/impl/impl.go](backend/plugins/gitlab/impl/impl.go) f
 - `PluginMigration`: MigrationScripts() for DB schema evolution
 - `PluginSource`: Connection(), Scope(), ScopeConfig()
 
+### Advanced Plugin Interfaces
+- `PluginInit`: Optional initialization hook with `Init(basicRes)` method for resource setup
+- `PluginOpenApiSpec`: Remote plugins can expose OpenAPI specs via `OpenApiSpec()` method
+- `PluginMetric`: For metrics plugins requiring `RequiredDataEntities()`, `IsProjectMetric()`, `RunAfter()`, `Settings()`
+- `DataSourcePluginBlueprintV200`: Project-aware pipeline generation with `MakeDataSourcePipelinePlanV200()` for cross-plugin scope mapping
+- `MetricPluginBlueprintV200`: Similar to DataSourcePluginBlueprintV200 for metric calculation plugins
+
+### Authentication Patterns
+Plugins can support multiple authentication methods via these interfaces:
+- `CacheableConnection`: Extends `ApiConnection` with `GetHash()` for connection caching
+- `MultiAuthenticator`: Base interface with `GetAuthMethod()` returning `BasicAuth`, `AccessToken`, or `AppKey`
+- `BasicAuthenticator`: Implement `GetBasicAuthenticator()` for HTTP Basic auth
+- `AccessTokenAuthenticator`: Implement `GetAccessTokenAuthenticator()` for Bearer token auth
+- `AppKeyAuthenticator`: Implement `GetAppKeyAuthenticator()` for API key/secret pairs
+- `PrepareApiClient`: Hook in connection for initialization (e.g., token refresh) via `PrepareApiClient(apiClient)`
+
+### Dynamic Models
+- `DynamicTabler` interface: For runtime-generated models with methods `Unwrap()`, `NewValue()`, `From()`, `To()`
+
 ### Subtask Pattern (Collector → Extractor → Converter)
 ```go
 // 1. Register subtask in tasks/register.go via init()
@@ -82,13 +101,18 @@ var CollectIssuesMeta = plugin.SubTaskMeta{
 make dep              # Install Go + Python dependencies
 make build            # Build plugins + server
 make dev              # Build + run server
-make godev            # Go-only dev (no Python plugins)
+make godev            # Go-only dev (no Python remote plugins)
 make unit-test        # Run all unit tests
 make e2e-test         # Run E2E tests
 
 # From backend/
 make swag             # Regenerate Swagger docs (required after API changes)
 make lint             # Run golangci-lint
+make mock             # Regenerate mocks from interfaces
+make migration-script-lint  # Validate migration script format
+make build-plugin-debug     # Build plugins with debug symbols (DEVLAKE_DEBUG=1)
+make build-pydevlake        # Install/sync Python plugin framework dependencies
+make e2e-test-go-plugins    # Run E2E tests for Go plugins only
 ```
 
 ### Running Locally
@@ -101,7 +125,7 @@ cd config-ui && yarn && yarn start                          # UI on :4000
 ## Testing
 
 ### Unit Tests
-Place `*_test.go` files alongside source. Use mocks from `backend/mocks/`.
+Place `*_test.go` files alongside source. Use mocks from `backend/mocks/`. Mocks are auto-generated via `make mock` from all interfaces in `core/` and `helpers/`.
 
 ### E2E Tests for Plugins
 Use CSV fixtures in `e2e/` directory. See [backend/test/helper/](backend/test/helper/) for the Go test client that can spin up an in-memory DevLake instance.
@@ -116,6 +140,9 @@ helper.ConnectLocalServer(t, &helper.LocalClientConfig{
 })
 ```
 
+### Model Validation
+Run `make migration-script-lint` from `backend/` to validate all migration scripts follow the correct format (`YYYYMMDD_description.go`).
+
 ## Python Plugins
 Located in `backend/python/plugins/`. Use Poetry for dependencies. See [backend/python/README.md](backend/python/README.md).
 
@@ -124,9 +151,14 @@ Located in `backend/python/plugins/`. Use Poetry for dependencies. See [backend/
 - Domain model IDs: Use `didgen.NewDomainIdGenerator` for consistent cross-plugin IDs
 - All plugins must be independent - no cross-plugin imports
 - Apache 2.0 license header required on all source files
+- Mocks are auto-generated: don't edit files in `backend/mocks/`, regenerate with `make mock`
+- Optional plugin interfaces (PluginInit, PluginOpenApiSpec, PluginMetric) should only be implemented if functionality is needed
+- Authentication: Use `CacheableConnection` interface if connection needs caching; implement appropriate `*Authenticator` for each auth method supported
 
 ## Common Pitfalls
 - Forgetting to add models to `GetTablesInfo()` fails `plugins/table_info_test.go`
-- Migration scripts must be added to `All()` in `register.go`
-- API changes require running `make swag` to update Swagger docs
+- Migration scripts must be added to `All()` in `register.go` AND follow `YYYYMMDD_description.go` naming (validate with `make migration-script-lint`)
+- API changes require running `make swag` to update Swagger docs (this runs `make mock` first)
 - Python plugins require `libgit2` for gitextractor functionality
+- New Plugin interfaces like `PluginInit` or `PluginOpenApiSpec` are optional - only implement if needed
+- Blueprint V2.0 implementation required for plugins that support project-aware scope mapping

TODO.md

@@ -0,0 +1,29 @@
+# TODO
+
+## Tech Debt
+
+### Migrate `q_dev` plugin from aws-sdk-go (v1) to aws-sdk-go-v2
+
+`github.com/aws/aws-sdk-go` (v1) is deprecated and reached end of support on
+July 31, 2025. `make lint` reports 11 `staticcheck SA1019` warnings for the
+`q_dev` plugin. These are currently non-blocking in `scripts-local/build.sh`
+(lint is a warning), but should be migrated to `aws-sdk-go-v2`.
+
+Affected files:
+- `backend/plugins/q_dev/tasks/identity_client.go`
+- `backend/plugins/q_dev/tasks/s3_client.go`
+- `backend/plugins/q_dev/tasks/s3_data_extractor.go`
+- `backend/plugins/q_dev/tasks/s3_file_collector.go`
+
+Required changes:
+- Replace `github.com/aws/aws-sdk-go/aws` and `.../aws/credentials` with the
+  v2 equivalents (`github.com/aws/aws-sdk-go-v2/aws`, `.../config`,
+  `.../credentials`).
+- Replace `.../aws/session` with v2 config loading
+  (`config.LoadDefaultConfig`).
+- Replace `.../service/identitystore` and `.../service/s3` with their v2
+  service clients (`aws-sdk-go-v2/service/identitystore`, `.../service/s3`).
+- Update `go.mod`/`go.sum` accordingly and remove the v1 dependency once unused.
+- Re-run `make lint` to confirm the `SA1019` warnings are gone, then restore
+  blocking lint in `scripts-local/build.sh`.
+

backend/.dockerignore

@@ -0,0 +1,24 @@
+# Build artifacts: regenerated inside the image (GOBIN=/app/bin + make build-plugin/build-server).
+# Shipping the host's pre-built bin/ (multiple GB of plugin .so files) bloats the
+# build context and the builder layer, which can exhaust the Docker VM and crash BuildKit.
+bin/
+
+# Generated mocks (regenerated via make mock inside the image)
+mocks/
+
+# Runtime logs
+logs/
+
+# Python build/venv artifacts (regenerated; Dockerfile.server reinstalls from
+# requirements.txt / pyproject and runs python/build.sh). Source under python/ is kept.
+**/.venv/
+**/__pycache__/
+**/*.pyc
+python/.devlake-python-build-root/
+
+# Test / coverage output
+*.out
+coverage.txt
+
+# OS / editor junk
+.DS_Store

backend/Dockerfile

@@ -50,7 +50,7 @@ RUN if [ "$(arch)" != "x86_64" ] ; then \
     fi
 
 RUN go install github.com/vektra/mockery/v2@v2.53.6
-RUN go install github.com/swaggo/swag/cmd/swag@v1.16.1
+RUN go install github.com/swaggo/swag/cmd/swag@v1.16.6
 
 COPY --from=debian-amd64 /usr/include /rootfs-amd64/usr/include
 COPY --from=debian-amd64 /usr/lib/x86_64-linux-gnu /rootfs-amd64/usr/lib/x86_64-linux-gnu
@@ -113,7 +113,7 @@ RUN cd /usr/local/deps/target/lib && \
     done
 
 
-FROM python:3.9-slim-bookworm as base
+FROM python:3.11-slim-bookworm as base
 
 RUN apt-get update && \
     apt-get install -y python3-dev python3-pip tar pkg-config curl libssh2-1 zlib1g libffi-dev default-libmysqlclient-dev libpq-dev tini git openssh-client corkscrew && \
@@ -181,4 +181,74 @@ USER devlake
 #add tini, prevent zombie process
 ENTRYPOINT ["/usr/bin/tini", "--"]
 
-CMD ["lake"]
+CMD ["lake"]    libssh2-1-dev \
+    libssl-dev \
+    zlib1g-dev
+
+# Install libgit2 for native platform only
+RUN mkdir -p /tmp/build && cd /tmp/build && \
+    wget https://github.com/libgit2/libgit2/archive/refs/tags/v1.3.0.tar.gz -O - | tar -xz && \
+    cd libgit2-1.3.0 && \
+    mkdir build && cd build && \
+    cmake .. -DBUILD_SHARED_LIBS=ON -DCMAKE_INSTALL_PREFIX=/usr/local && \
+    make -j$(nproc) install && \
+    ldconfig
+
+RUN go install github.com/vektra/mockery/v2@v2.53.6
+RUN go install github.com/swaggo/swag/cmd/swag@v1.16.6
+
+WORKDIR /app
+
+# Cache Go module downloads in a separate layer (only re-runs when go.mod/go.sum change)
+COPY go.mod go.sum ./
+RUN --mount=type=cache,target=/go/pkg/mod \
+    go mod download
+
+# Now copy the rest of the source
+COPY . /app
+ENV GOBIN=/app/bin
+
+ARG TAG=
+ARG SHA=
+
+# Generate mocks and swagger (use build cache to speed up swag)
+RUN --mount=type=cache,target=/root/.cache/go-build \
+    --mount=type=cache,target=/go/pkg/mod \
+    make mock
+RUN --mount=type=cache,target=/root/.cache/go-build \
+    --mount=type=cache,target=/go/pkg/mod \
+    make swag
+
+# Build plugins (parallel, with persistent Go build cache)
+RUN --mount=type=cache,target=/root/.cache/go-build \
+    --mount=type=cache,target=/go/pkg/mod \
+    make build-plugin
+
+# Build server
+RUN --mount=type=cache,target=/root/.cache/go-build \
+    --mount=type=cache,target=/go/pkg/mod \
+    VERSION=${TAG}@${SHA} make build-server
+
+FROM debian:bookworm-slim
+
+RUN apt-get update && apt-get install -y \
+    libssh2-1 \
+    libssl3 \
+    ca-certificates \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy libgit2
+COPY --from=builder /usr/local/lib/libgit2.so* /usr/local/lib/
+RUN ldconfig
+
+WORKDIR /app
+
+COPY --from=builder /app/bin ./bin
+
+# Create empty python plugins dir to avoid startup error
+RUN mkdir -p python/plugins
+
+ENV PORT=8080
+EXPOSE 8080
+
+CMD ["./bin/lake"]