Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions internal/import/d1/doctor.go
Original file line number Diff line number Diff line change
Expand Up @@ -222,3 +222,13 @@ func FindSQLite3() (string, error) {
}
return path, nil
}

// sqliteCLIArgs builds sqlite3 arguments that isolate the run from user
// configuration. The sqlite3 shell loads ~/.sqliterc even for one-shot
// invocations (found via the passwd database, so $HOME overrides don't help);
// settings like `.headers on` or `.mode column` change the output format and
// break parsing. -init with an empty file suppresses the rc file, -batch and
// -noheader force plain list output.
func sqliteCLIArgs(dbPath string, args ...string) []string {
return append([]string{"-batch", "-noheader", "-init", os.DevNull, dbPath}, args...)
}
2 changes: 1 addition & 1 deletion internal/import/d1/sqlite_load.go
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,7 @@ func loadSQLiteDumpChunked(ctx context.Context, sqlite3, dumpPath, sqlitePath st
chunkFile = nil

readPath := strings.ReplaceAll(chunkPath, "'", "''")
cmd := execabs.CommandContext(ctx, sqlite3, sqlitePath, fmt.Sprintf(".read %s", readPath))
cmd := execabs.CommandContext(ctx, sqlite3, sqliteCLIArgs(sqlitePath, fmt.Sprintf(".read %s", readPath))...)
var stderr bytes.Buffer
cmd.Stdout = io.Discard
cmd.Stderr = &stderr
Expand Down
18 changes: 13 additions & 5 deletions internal/import/d1/verify.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,11 @@ package d1

import (
"context"
"errors"
"fmt"
"os/exec"
"strconv"
"strings"
"time"

"github.com/planetscale/cli/internal/postgres"
Expand Down Expand Up @@ -193,15 +197,19 @@ func countSQLiteRowsWithProgress(ctx context.Context, opts VerifyOptions, sqlite
Total: len(tables),
Detail: table + " (sqlite)",
})
query := fmt.Sprintf("SELECT COUNT(*) FROM %q;", table)
cmd := execabs.CommandContext(ctx, sqlite3, sqlitePath, query)
query := fmt.Sprintf("SELECT COUNT(*) FROM %s;", quoteSQLiteIdentifier(table))
cmd := execabs.CommandContext(ctx, sqlite3, sqliteCLIArgs(sqlitePath, query)...)
out, err := cmd.Output()
if err != nil {
var ee *exec.ExitError
if errors.As(err, &ee) && len(ee.Stderr) > 0 {
return nil, fmt.Errorf("sqlite count %s: %w: %s", table, err, truncateLoadError(strings.TrimSpace(string(ee.Stderr)), 200))
}
return nil, fmt.Errorf("sqlite count %s: %w", table, err)
}
var count int64
if _, err := fmt.Sscanf(string(out), "%d", &count); err != nil {
return nil, err
count, err := strconv.ParseInt(strings.TrimSpace(string(out)), 10, 64)
if err != nil {
return nil, fmt.Errorf("sqlite count %s: unexpected output %q", table, truncateLoadError(string(out), 120))
}
counts[table] = count
}
Expand Down
54 changes: 36 additions & 18 deletions internal/import/d1/verify_checks.go
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
package d1

import (
"bytes"
"context"
"database/sql"
"encoding/hex"
"encoding/json"
"fmt"
"strconv"
"strings"
"unicode/utf8"

Expand Down Expand Up @@ -261,9 +263,11 @@ func querySQLiteDistribution(ctx context.Context, sqlitePath, query string) (boo
if parts[i] == "" || parts[i] == "NULL" {
continue
}
if _, err := fmt.Sscanf(parts[i], "%d", ptr); err != nil {
return booleanDistribution{}, err
n, err := strconv.ParseInt(parts[i], 10, 64)
if err != nil {
return booleanDistribution{}, fmt.Errorf("boolean distribution: unexpected count %q", parts[i])
}
*ptr = n
}
return dist, nil
}
Expand Down Expand Up @@ -365,9 +369,9 @@ func shouldFingerprintPKSum(table TableSchema, pkCol string, all []TableSchema,
func tableFingerprintFromSQLite(ctx context.Context, sqlitePath string, table TableSchema, pkCol string, all []TableSchema, coerceCtx *TypeCoercionContext) (tableFingerprint, error) {
var query string
if pkCol != "" && shouldFingerprintPKSum(table, pkCol, all, coerceCtx) {
query = fmt.Sprintf(`SELECT COUNT(*), COALESCE(CAST(SUM(CAST(%q AS INTEGER)) AS TEXT), '0') FROM %q;`, pkCol, table.Name)
query = fmt.Sprintf(`SELECT COUNT(*), COALESCE(CAST(SUM(CAST(%s AS INTEGER)) AS TEXT), '0') FROM %s;`, quoteSQLiteIdentifier(pkCol), quoteSQLiteIdentifier(table.Name))
} else {
query = fmt.Sprintf(`SELECT COUNT(*), '0' FROM %q;`, table.Name)
query = fmt.Sprintf(`SELECT COUNT(*), '0' FROM %s;`, quoteSQLiteIdentifier(table.Name))
}
sqlite3, err := FindSQLite3()
if err != nil {
Expand All @@ -382,9 +386,11 @@ func tableFingerprintFromSQLite(ctx context.Context, sqlitePath string, table Ta
if len(fields) < 2 {
return tableFingerprint{}, fmt.Errorf("sqlite fingerprint %s: unexpected output %q", table.Name, string(out))
}
if _, err := fmt.Sscanf(fields[0], "%d", &fp.RowCount); err != nil {
return tableFingerprint{}, fmt.Errorf("sqlite fingerprint %s row count: %w", table.Name, err)
rowCount, err := strconv.ParseInt(fields[0], 10, 64)
if err != nil {
return tableFingerprint{}, fmt.Errorf("sqlite fingerprint %s: unexpected row count %q", table.Name, fields[0])
}
fp.RowCount = rowCount
fp.IDSum = fields[1]
return fp, nil
}
Expand Down Expand Up @@ -472,7 +478,7 @@ func samplePrimaryKeys(ctx context.Context, sqlitePath, table, pkCol string, lim
if err != nil {
return nil, err
}
query := fmt.Sprintf(`SELECT %q FROM %q ORDER BY %q LIMIT %d;`, pkCol, table, pkCol, limit)
query := fmt.Sprintf(`SELECT %s FROM %s ORDER BY %s LIMIT %d;`, quoteSQLiteIdentifier(pkCol), quoteSQLiteIdentifier(table), quoteSQLiteIdentifier(pkCol), limit)
out, err := runSQLiteQuery(ctx, sqlite3, sqlitePath, query)
if err != nil {
return nil, err
Expand All @@ -489,19 +495,20 @@ func samplePrimaryKeys(ctx context.Context, sqlitePath, table, pkCol string, lim
}

func sqliteSignatureColumnExpr(col ColumnSchema, table TableSchema, coerceCtx *TypeCoercionContext) string {
name := quoteSQLiteIdentifier(col.Name)
if isBooleanLikeColumn(col, table, coerceCtx) {
return fmt.Sprintf(`CASE WHEN %q IN (1, '1') THEN '1' WHEN %q IN (0, '0') THEN '0' ELSE '' END`, col.Name, col.Name)
return fmt.Sprintf(`CASE WHEN %s IN (1, '1') THEN '1' WHEN %s IN (0, '0') THEN '0' ELSE '' END`, name, name)
}
if isJSONText(col) && coerceCtx != nil && samplesAllowJSON(table.Name, col.Name, coerceCtx) {
return fmt.Sprintf(`COALESCE(json(%q), CAST(%q AS TEXT), '')`, col.Name, col.Name)
return fmt.Sprintf(`COALESCE(json(%s), CAST(%s AS TEXT), '')`, name, name)
}
if isBlobColumn(col) {
return fmt.Sprintf(`COALESCE(hex(%q), '')`, col.Name)
return fmt.Sprintf(`COALESCE(hex(%s), '')`, name)
}
if isTimestampText(col) && coerceCtx != nil && samplesAllowTimestamp(table.Name, col.Name, coerceCtx) {
return fmt.Sprintf(`COALESCE(strftime('%%Y-%%m-%%dT%%H:%%M:%%SZ', %q), COALESCE(CAST(%q AS TEXT), ''))`, col.Name, col.Name)
return fmt.Sprintf(`COALESCE(strftime('%%Y-%%m-%%dT%%H:%%M:%%SZ', %s), COALESCE(CAST(%s AS TEXT), ''))`, name, name)
}
return fmt.Sprintf(`COALESCE(CAST(%q AS TEXT), '')`, col.Name)
return fmt.Sprintf(`COALESCE(CAST(%s AS TEXT), '')`, name)
}

func postgresSignatureColumnExpr(col ColumnSchema, table TableSchema, all []TableSchema, coerceCtx *TypeCoercionContext) string {
Expand Down Expand Up @@ -693,6 +700,13 @@ func postgresRowSignature(ctx context.Context, db *sql.DB, table TableSchema, pk
return sig.String, nil
}

// quoteSQLiteIdentifier double-quotes an identifier, escaping embedded double
// quotes. Go's %q is not valid SQL quoting for names containing quotes or
// backslashes.
func quoteSQLiteIdentifier(name string) string {
return `"` + strings.ReplaceAll(name, `"`, `""`) + `"`
}

func sqliteLiteral(val string) string {
if val != "" && isSQLiteIntegerLiteral(val) {
return val
Expand All @@ -715,11 +729,15 @@ func isSQLiteIntegerLiteral(val string) bool {
return true
}

// runSQLiteQuery returns sqlite3's stdout. Stderr is kept out of the result
// (warnings there would corrupt parsing) and reported only on failure.
func runSQLiteQuery(ctx context.Context, sqlite3, sqlitePath, query string) ([]byte, error) {
cmd := execabs.CommandContext(ctx, sqlite3, sqlitePath, query)
out, err := cmd.CombinedOutput()
if err != nil {
return nil, fmt.Errorf("%w: %s", err, strings.TrimSpace(string(out)))
}
return out, nil
cmd := execabs.CommandContext(ctx, sqlite3, sqliteCLIArgs(sqlitePath, query)...)
var stdout, stderr bytes.Buffer
cmd.Stdout = &stdout
cmd.Stderr = &stderr
if err := cmd.Run(); err != nil {
return nil, fmt.Errorf("%w: %s", err, strings.TrimSpace(stderr.String()))
}
return stdout.Bytes(), nil
}