GitSecrets endpoints
This commit is contained in:
392
modules/secretscan/patterns.go
Normal file
392
modules/secretscan/patterns.go
Normal file
@@ -0,0 +1,392 @@
|
||||
// Copyright 2026 MarketAlly. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package secretscan
|
||||
|
||||
import "regexp"
|
||||
|
||||
// Severity represents the severity level of a detected secret
|
||||
type Severity string
|
||||
|
||||
const (
|
||||
SeverityCritical Severity = "critical"
|
||||
SeverityHigh Severity = "high"
|
||||
SeverityMedium Severity = "medium"
|
||||
SeverityLow Severity = "low"
|
||||
)
|
||||
|
||||
// Category represents the category of a detected secret
|
||||
type Category string
|
||||
|
||||
const (
|
||||
CategoryAPIKey Category = "api-key"
|
||||
CategoryPrivateKey Category = "private-key"
|
||||
CategoryPassword Category = "password"
|
||||
CategoryToken Category = "token"
|
||||
CategoryCertificate Category = "certificate"
|
||||
CategoryConnectionString Category = "connection-string"
|
||||
CategoryCredential Category = "credential"
|
||||
CategorySecret Category = "secret"
|
||||
)
|
||||
|
||||
// Pattern represents a secret detection pattern
|
||||
type Pattern struct {
|
||||
ID string
|
||||
Name string
|
||||
Description string
|
||||
Regex *regexp.Regexp
|
||||
Severity Severity
|
||||
Category Category
|
||||
FalsePositiveRegexes []*regexp.Regexp
|
||||
}
|
||||
|
||||
// builtinPatterns contains all built-in secret detection patterns
|
||||
var builtinPatterns = []Pattern{
|
||||
// ============================================
|
||||
// PRIVATE KEYS (Critical)
|
||||
// ============================================
|
||||
{
|
||||
ID: "private-key-rsa",
|
||||
Name: "RSA Private Key",
|
||||
Description: "RSA private key in PEM format",
|
||||
Regex: regexp.MustCompile(`-----BEGIN (?:RSA )?PRIVATE KEY-----[\s\S]*?-----END (?:RSA )?PRIVATE KEY-----`),
|
||||
Severity: SeverityCritical,
|
||||
Category: CategoryPrivateKey,
|
||||
},
|
||||
{
|
||||
ID: "private-key-openssh",
|
||||
Name: "OpenSSH Private Key",
|
||||
Description: "OpenSSH private key",
|
||||
Regex: regexp.MustCompile(`-----BEGIN OPENSSH PRIVATE KEY-----[\s\S]*?-----END OPENSSH PRIVATE KEY-----`),
|
||||
Severity: SeverityCritical,
|
||||
Category: CategoryPrivateKey,
|
||||
},
|
||||
{
|
||||
ID: "private-key-ec",
|
||||
Name: "EC Private Key",
|
||||
Description: "Elliptic Curve private key",
|
||||
Regex: regexp.MustCompile(`-----BEGIN EC PRIVATE KEY-----[\s\S]*?-----END EC PRIVATE KEY-----`),
|
||||
Severity: SeverityCritical,
|
||||
Category: CategoryPrivateKey,
|
||||
},
|
||||
{
|
||||
ID: "private-key-dsa",
|
||||
Name: "DSA Private Key",
|
||||
Description: "DSA private key",
|
||||
Regex: regexp.MustCompile(`-----BEGIN DSA PRIVATE KEY-----[\s\S]*?-----END DSA PRIVATE KEY-----`),
|
||||
Severity: SeverityCritical,
|
||||
Category: CategoryPrivateKey,
|
||||
},
|
||||
{
|
||||
ID: "private-key-pgp",
|
||||
Name: "PGP Private Key",
|
||||
Description: "PGP/GPG private key block",
|
||||
Regex: regexp.MustCompile(`-----BEGIN PGP PRIVATE KEY BLOCK-----[\s\S]*?-----END PGP PRIVATE KEY BLOCK-----`),
|
||||
Severity: SeverityCritical,
|
||||
Category: CategoryPrivateKey,
|
||||
},
|
||||
|
||||
// ============================================
|
||||
// AWS (Critical)
|
||||
// ============================================
|
||||
{
|
||||
ID: "aws-access-key",
|
||||
Name: "AWS Access Key ID",
|
||||
Description: "Amazon Web Services access key identifier",
|
||||
Regex: regexp.MustCompile(`\b(A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}\b`),
|
||||
Severity: SeverityCritical,
|
||||
Category: CategoryAPIKey,
|
||||
},
|
||||
{
|
||||
ID: "aws-mws-key",
|
||||
Name: "AWS MWS Key",
|
||||
Description: "Amazon Marketplace Web Service key",
|
||||
Regex: regexp.MustCompile(`(?i)amzn\.mws\.[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}`),
|
||||
Severity: SeverityCritical,
|
||||
Category: CategoryAPIKey,
|
||||
},
|
||||
|
||||
// ============================================
|
||||
// GOOGLE / GCP (Critical)
|
||||
// ============================================
|
||||
{
|
||||
ID: "gcp-api-key",
|
||||
Name: "Google API Key",
|
||||
Description: "Google Cloud Platform API key",
|
||||
Regex: regexp.MustCompile(`AIza[0-9A-Za-z_-]{35}`),
|
||||
Severity: SeverityCritical,
|
||||
Category: CategoryAPIKey,
|
||||
},
|
||||
{
|
||||
ID: "gcp-service-account",
|
||||
Name: "GCP Service Account",
|
||||
Description: "Google Cloud service account key file",
|
||||
Regex: regexp.MustCompile(`"type"\s*:\s*"service_account"[\s\S]*?"private_key"`),
|
||||
Severity: SeverityCritical,
|
||||
Category: CategoryPrivateKey,
|
||||
},
|
||||
|
||||
// ============================================
|
||||
// AZURE (Critical)
|
||||
// ============================================
|
||||
{
|
||||
ID: "azure-connection-string",
|
||||
Name: "Azure Connection String",
|
||||
Description: "Azure service connection string",
|
||||
Regex: regexp.MustCompile(`(?i)DefaultEndpointsProtocol=https?;AccountName=[^;]+;AccountKey=[A-Za-z0-9/+=]+`),
|
||||
Severity: SeverityCritical,
|
||||
Category: CategoryConnectionString,
|
||||
},
|
||||
|
||||
// ============================================
|
||||
// GITHUB (High)
|
||||
// ============================================
|
||||
{
|
||||
ID: "github-token",
|
||||
Name: "GitHub Token",
|
||||
Description: "GitHub personal access token or OAuth token",
|
||||
Regex: regexp.MustCompile(`\b(ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9]{36,}\b`),
|
||||
Severity: SeverityHigh,
|
||||
Category: CategoryToken,
|
||||
},
|
||||
|
||||
// ============================================
|
||||
// GITLAB (High)
|
||||
// ============================================
|
||||
{
|
||||
ID: "gitlab-token",
|
||||
Name: "GitLab Token",
|
||||
Description: "GitLab personal access token",
|
||||
Regex: regexp.MustCompile(`\bglpat-[A-Za-z0-9_-]{20,}\b`),
|
||||
Severity: SeverityHigh,
|
||||
Category: CategoryToken,
|
||||
},
|
||||
|
||||
// ============================================
|
||||
// STRIPE (Critical)
|
||||
// ============================================
|
||||
{
|
||||
ID: "stripe-secret-key",
|
||||
Name: "Stripe Secret Key",
|
||||
Description: "Stripe API secret key",
|
||||
Regex: regexp.MustCompile(`\bsk_live_[0-9a-zA-Z]{24,}\b`),
|
||||
Severity: SeverityCritical,
|
||||
Category: CategoryAPIKey,
|
||||
},
|
||||
{
|
||||
ID: "stripe-test-key",
|
||||
Name: "Stripe Test Key",
|
||||
Description: "Stripe API test key (still sensitive)",
|
||||
Regex: regexp.MustCompile(`\bsk_test_[0-9a-zA-Z]{24,}\b`),
|
||||
Severity: SeverityMedium,
|
||||
Category: CategoryAPIKey,
|
||||
},
|
||||
|
||||
// ============================================
|
||||
// SLACK (High)
|
||||
// ============================================
|
||||
{
|
||||
ID: "slack-token",
|
||||
Name: "Slack Token",
|
||||
Description: "Slack API token",
|
||||
Regex: regexp.MustCompile(`\bxox[baprs]-[0-9]{10,13}-[0-9]{10,13}[a-zA-Z0-9-]*\b`),
|
||||
Severity: SeverityHigh,
|
||||
Category: CategoryToken,
|
||||
},
|
||||
{
|
||||
ID: "slack-webhook",
|
||||
Name: "Slack Webhook URL",
|
||||
Description: "Slack incoming webhook URL",
|
||||
Regex: regexp.MustCompile(`https://hooks\.slack\.com/services/T[A-Z0-9]+/B[A-Z0-9]+/[A-Za-z0-9]+`),
|
||||
Severity: SeverityHigh,
|
||||
Category: CategoryCredential,
|
||||
},
|
||||
|
||||
// ============================================
|
||||
// TWILIO (High)
|
||||
// ============================================
|
||||
{
|
||||
ID: "twilio-api-key",
|
||||
Name: "Twilio API Key",
|
||||
Description: "Twilio API key SID",
|
||||
Regex: regexp.MustCompile(`\bSK[0-9a-fA-F]{32}\b`),
|
||||
Severity: SeverityHigh,
|
||||
Category: CategoryAPIKey,
|
||||
},
|
||||
|
||||
// ============================================
|
||||
// SENDGRID (High)
|
||||
// ============================================
|
||||
{
|
||||
ID: "sendgrid-api-key",
|
||||
Name: "SendGrid API Key",
|
||||
Description: "SendGrid email API key",
|
||||
Regex: regexp.MustCompile(`\bSG\.[A-Za-z0-9_-]{22}\.[A-Za-z0-9_-]{43}\b`),
|
||||
Severity: SeverityHigh,
|
||||
Category: CategoryAPIKey,
|
||||
},
|
||||
|
||||
// ============================================
|
||||
// NPM (High)
|
||||
// ============================================
|
||||
{
|
||||
ID: "npm-token",
|
||||
Name: "NPM Access Token",
|
||||
Description: "NPM registry access token",
|
||||
Regex: regexp.MustCompile(`\bnpm_[A-Za-z0-9]{36}\b`),
|
||||
Severity: SeverityHigh,
|
||||
Category: CategoryToken,
|
||||
},
|
||||
|
||||
// ============================================
|
||||
// DATABASE CONNECTION STRINGS (Critical)
|
||||
// ============================================
|
||||
{
|
||||
ID: "postgres-uri",
|
||||
Name: "PostgreSQL Connection URI",
|
||||
Description: "PostgreSQL connection string with credentials",
|
||||
Regex: regexp.MustCompile(`(?i)postgres(?:ql)?://[^:]+:[^@]+@[^/]+/[^\s"']+`),
|
||||
Severity: SeverityCritical,
|
||||
Category: CategoryConnectionString,
|
||||
},
|
||||
{
|
||||
ID: "mysql-uri",
|
||||
Name: "MySQL Connection URI",
|
||||
Description: "MySQL connection string with credentials",
|
||||
Regex: regexp.MustCompile(`(?i)mysql://[^:]+:[^@]+@[^/]+/[^\s"']+`),
|
||||
Severity: SeverityCritical,
|
||||
Category: CategoryConnectionString,
|
||||
},
|
||||
{
|
||||
ID: "mongodb-uri",
|
||||
Name: "MongoDB Connection URI",
|
||||
Description: "MongoDB connection string with credentials",
|
||||
Regex: regexp.MustCompile(`(?i)mongodb(?:\+srv)?://[^:]+:[^@]+@[^/]+`),
|
||||
Severity: SeverityCritical,
|
||||
Category: CategoryConnectionString,
|
||||
},
|
||||
|
||||
// ============================================
|
||||
// GENERIC PASSWORDS (High)
|
||||
// ============================================
|
||||
{
|
||||
ID: "password-assignment",
|
||||
Name: "Password Assignment",
|
||||
Description: "Hardcoded password in code",
|
||||
Regex: regexp.MustCompile(`(?i)(?:password|passwd|pwd|secret|api_?key|auth_?token|access_?token)\s*[:=]\s*['"][^'"]{8,}['"]`),
|
||||
Severity: SeverityHigh,
|
||||
Category: CategoryPassword,
|
||||
FalsePositiveRegexes: []*regexp.Regexp{
|
||||
regexp.MustCompile(`\$\{.*\}`), // Template variables
|
||||
regexp.MustCompile(`process\.env`), // Environment references
|
||||
regexp.MustCompile(`\{\{.*\}\}`), // Handlebars/mustache
|
||||
regexp.MustCompile(`<.*>`), // Placeholder text
|
||||
regexp.MustCompile(`(?i)example`), // Example text
|
||||
regexp.MustCompile(`(?i)placeholder`),
|
||||
},
|
||||
},
|
||||
|
||||
// ============================================
|
||||
// JWT (Medium)
|
||||
// ============================================
|
||||
{
|
||||
ID: "jwt-token",
|
||||
Name: "JSON Web Token",
|
||||
Description: "JWT token (may contain sensitive claims)",
|
||||
Regex: regexp.MustCompile(`\beyJ[A-Za-z0-9_-]*\.eyJ[A-Za-z0-9_-]*\.[A-Za-z0-9_-]*`),
|
||||
Severity: SeverityMedium,
|
||||
Category: CategoryToken,
|
||||
},
|
||||
|
||||
// ============================================
|
||||
// DISCORD (High)
|
||||
// ============================================
|
||||
{
|
||||
ID: "discord-token",
|
||||
Name: "Discord Bot Token",
|
||||
Description: "Discord bot or user token",
|
||||
Regex: regexp.MustCompile(`[MN][A-Za-z\d]{23,}\.[\w-]{6}\.[\w-]{27}`),
|
||||
Severity: SeverityHigh,
|
||||
Category: CategoryToken,
|
||||
},
|
||||
{
|
||||
ID: "discord-webhook",
|
||||
Name: "Discord Webhook URL",
|
||||
Description: "Discord webhook URL",
|
||||
Regex: regexp.MustCompile(`https://discord(?:app)?\.com/api/webhooks/[0-9]+/[A-Za-z0-9_-]+`),
|
||||
Severity: SeverityHigh,
|
||||
Category: CategoryCredential,
|
||||
},
|
||||
|
||||
// ============================================
|
||||
// TELEGRAM (High)
|
||||
// ============================================
|
||||
{
|
||||
ID: "telegram-bot-token",
|
||||
Name: "Telegram Bot Token",
|
||||
Description: "Telegram Bot API token",
|
||||
Regex: regexp.MustCompile(`\b[0-9]{8,10}:[A-Za-z0-9_-]{35}\b`),
|
||||
Severity: SeverityHigh,
|
||||
Category: CategoryToken,
|
||||
},
|
||||
|
||||
// ============================================
|
||||
// SHOPIFY (High)
|
||||
// ============================================
|
||||
{
|
||||
ID: "shopify-token",
|
||||
Name: "Shopify Access Token",
|
||||
Description: "Shopify private app or custom app token",
|
||||
Regex: regexp.MustCompile(`shpat_[a-fA-F0-9]{32}`),
|
||||
Severity: SeverityHigh,
|
||||
Category: CategoryToken,
|
||||
},
|
||||
|
||||
// ============================================
|
||||
// HEROKU (High)
|
||||
// ============================================
|
||||
{
|
||||
ID: "heroku-api-key",
|
||||
Name: "Heroku API Key",
|
||||
Description: "Heroku platform API key",
|
||||
Regex: regexp.MustCompile(`(?i)heroku.*['\"][0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}['\"]`),
|
||||
Severity: SeverityHigh,
|
||||
Category: CategoryAPIKey,
|
||||
},
|
||||
}
|
||||
|
||||
// genericFalsePositivePatterns are patterns that indicate false positives across all detections
|
||||
var genericFalsePositivePatterns = []*regexp.Regexp{
|
||||
regexp.MustCompile(`^[xX]+$`), // All x's (placeholder)
|
||||
regexp.MustCompile(`^[0]+$`), // All zeros
|
||||
regexp.MustCompile(`(?i)example`), // Contains "example"
|
||||
regexp.MustCompile(`(?i)sample`), // Contains "sample"
|
||||
regexp.MustCompile(`(?i)dummy`), // Contains "dummy"
|
||||
regexp.MustCompile(`(?i)placeholder`), // Contains "placeholder"
|
||||
regexp.MustCompile(`(?i)your[_-]?(api[_-]?)?key`), // "your_key", "your_api_key", etc.
|
||||
regexp.MustCompile(`__[A-Z_]+__`), // Python dunder-like placeholders
|
||||
}
|
||||
|
||||
// GetBuiltinPatterns returns all built-in secret detection patterns
|
||||
func GetBuiltinPatterns() []Pattern {
|
||||
return builtinPatterns
|
||||
}
|
||||
|
||||
// IsFalsePositive checks if a match is a false positive
|
||||
func IsFalsePositive(match string, pattern *Pattern) bool {
|
||||
// Check pattern-specific false positives
|
||||
for _, fp := range pattern.FalsePositiveRegexes {
|
||||
if fp.MatchString(match) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
// Check generic false positives
|
||||
for _, fp := range genericFalsePositivePatterns {
|
||||
if fp.MatchString(match) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
428
modules/secretscan/scanner.go
Normal file
428
modules/secretscan/scanner.go
Normal file
@@ -0,0 +1,428 @@
|
||||
// Copyright 2026 MarketAlly. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package secretscan
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"code.gitea.io/gitea/modules/git"
|
||||
"code.gitea.io/gitea/modules/git/gitcmd"
|
||||
"code.gitea.io/gitea/modules/log"
|
||||
"code.gitea.io/gitea/modules/setting"
|
||||
)
|
||||
|
||||
// DetectedSecret represents a secret found during scanning
|
||||
type DetectedSecret struct {
|
||||
PatternID string `json:"pattern_id"`
|
||||
PatternName string `json:"pattern_name"`
|
||||
Category Category `json:"category"`
|
||||
Severity Severity `json:"severity"`
|
||||
FilePath string `json:"file_path"`
|
||||
LineNumber int `json:"line_number"`
|
||||
MatchedText string `json:"matched_text"`
|
||||
MaskedText string `json:"masked_text"`
|
||||
CommitSHA string `json:"commit_sha,omitempty"`
|
||||
}
|
||||
|
||||
// ScanResult contains the results of a secret scan
|
||||
type ScanResult struct {
|
||||
Secrets []DetectedSecret `json:"secrets"`
|
||||
ScannedFiles int `json:"scanned_files"`
|
||||
ScanDuration time.Duration `json:"scan_duration"`
|
||||
Blocked bool `json:"blocked"`
|
||||
Message string `json:"message,omitempty"`
|
||||
}
|
||||
|
||||
// Scanner scans content for secrets
|
||||
type Scanner struct {
|
||||
patterns []Pattern
|
||||
ignoredFiles []string
|
||||
}
|
||||
|
||||
// NewScanner creates a new secret scanner
|
||||
func NewScanner() *Scanner {
|
||||
return &Scanner{
|
||||
patterns: GetBuiltinPatterns(),
|
||||
ignoredFiles: getDefaultIgnoredFiles(),
|
||||
}
|
||||
}
|
||||
|
||||
// getDefaultIgnoredFiles returns the default list of ignored file patterns
|
||||
func getDefaultIgnoredFiles() []string {
|
||||
return []string{
|
||||
// Lock files
|
||||
"package-lock.json",
|
||||
"yarn.lock",
|
||||
"pnpm-lock.yaml",
|
||||
"go.sum",
|
||||
"Cargo.lock",
|
||||
"Gemfile.lock",
|
||||
"poetry.lock",
|
||||
"composer.lock",
|
||||
// Documentation
|
||||
"*.md",
|
||||
"*.txt",
|
||||
"LICENSE*",
|
||||
"CHANGELOG*",
|
||||
// Binary files (won't be in diff anyway but good to have)
|
||||
"*.exe", "*.dll", "*.so", "*.dylib",
|
||||
"*.png", "*.jpg", "*.jpeg", "*.gif", "*.ico",
|
||||
"*.zip", "*.tar", "*.gz", "*.rar",
|
||||
"*.pdf", "*.doc", "*.docx",
|
||||
// Test fixtures
|
||||
"**/testdata/*",
|
||||
"**/fixtures/*",
|
||||
"**/__mocks__/*",
|
||||
}
|
||||
}
|
||||
|
||||
// shouldIgnoreFile checks if a file should be ignored during scanning
|
||||
func (s *Scanner) shouldIgnoreFile(filePath string) bool {
|
||||
fileName := filepath.Base(filePath)
|
||||
|
||||
for _, pattern := range s.ignoredFiles {
|
||||
// Check exact match
|
||||
if pattern == fileName {
|
||||
return true
|
||||
}
|
||||
|
||||
// Check glob pattern
|
||||
if matched, _ := filepath.Match(pattern, fileName); matched {
|
||||
return true
|
||||
}
|
||||
|
||||
// Check path pattern
|
||||
if matched, _ := filepath.Match(pattern, filePath); matched {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// ScanContent scans text content for secrets
|
||||
func (s *Scanner) ScanContent(content, filePath string) []DetectedSecret {
|
||||
var secrets []DetectedSecret
|
||||
|
||||
if s.shouldIgnoreFile(filePath) {
|
||||
return secrets
|
||||
}
|
||||
|
||||
lines := strings.Split(content, "\n")
|
||||
|
||||
for _, pattern := range s.patterns {
|
||||
matches := pattern.Regex.FindAllStringIndex(content, -1)
|
||||
for _, match := range matches {
|
||||
matchedText := content[match[0]:match[1]]
|
||||
|
||||
// Skip false positives
|
||||
if IsFalsePositive(matchedText, &pattern) {
|
||||
continue
|
||||
}
|
||||
|
||||
// Calculate line number
|
||||
lineNumber := 1
|
||||
for _, c := range content[:match[0]] {
|
||||
if c == '\n' {
|
||||
lineNumber++
|
||||
}
|
||||
}
|
||||
|
||||
// Get context (the line)
|
||||
contextLine := ""
|
||||
if lineNumber > 0 && lineNumber <= len(lines) {
|
||||
contextLine = lines[lineNumber-1]
|
||||
}
|
||||
_ = contextLine
|
||||
|
||||
secrets = append(secrets, DetectedSecret{
|
||||
PatternID: pattern.ID,
|
||||
PatternName: pattern.Name,
|
||||
Category: pattern.Category,
|
||||
Severity: pattern.Severity,
|
||||
FilePath: filePath,
|
||||
LineNumber: lineNumber,
|
||||
MatchedText: matchedText,
|
||||
MaskedText: maskSecret(matchedText),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return secrets
|
||||
}
|
||||
|
||||
// ScanDiff scans a git diff for secrets (only added lines)
|
||||
func (s *Scanner) ScanDiff(diff string) []DetectedSecret {
|
||||
var secrets []DetectedSecret
|
||||
|
||||
lines := strings.Split(diff, "\n")
|
||||
currentFile := ""
|
||||
lineNumber := 0
|
||||
|
||||
for _, line := range lines {
|
||||
// Track current file from diff header
|
||||
if strings.HasPrefix(line, "+++ b/") {
|
||||
currentFile = strings.TrimPrefix(line, "+++ b/")
|
||||
lineNumber = 0
|
||||
continue
|
||||
}
|
||||
|
||||
// Track line numbers from hunk header
|
||||
if strings.HasPrefix(line, "@@") {
|
||||
// Parse @@ -x,y +a,b @@
|
||||
parts := strings.Split(line, " ")
|
||||
for _, part := range parts {
|
||||
if strings.HasPrefix(part, "+") && part != "+++" {
|
||||
fmt.Sscanf(part, "+%d", &lineNumber)
|
||||
lineNumber-- // Will be incremented below
|
||||
break
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
// Only scan added lines (starting with +)
|
||||
if strings.HasPrefix(line, "+") && !strings.HasPrefix(line, "+++") {
|
||||
lineNumber++
|
||||
content := line[1:] // Remove the + prefix
|
||||
|
||||
if s.shouldIgnoreFile(currentFile) {
|
||||
continue
|
||||
}
|
||||
|
||||
lineSecrets := s.ScanContent(content, currentFile)
|
||||
for i := range lineSecrets {
|
||||
lineSecrets[i].LineNumber = lineNumber
|
||||
}
|
||||
secrets = append(secrets, lineSecrets...)
|
||||
} else if !strings.HasPrefix(line, "-") {
|
||||
// Context line (no prefix) - increment line number
|
||||
lineNumber++
|
||||
}
|
||||
}
|
||||
|
||||
return secrets
|
||||
}
|
||||
|
||||
// ScanCommitRange scans all commits in a range for secrets
|
||||
func (s *Scanner) ScanCommitRange(ctx context.Context, repo *git.Repository, oldCommitID, newCommitID string) (*ScanResult, error) {
|
||||
startTime := time.Now()
|
||||
result := &ScanResult{
|
||||
Secrets: []DetectedSecret{},
|
||||
}
|
||||
|
||||
// Get object format to check for empty commit ID
|
||||
objectFormat, err := repo.GetObjectFormat()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get object format: %w", err)
|
||||
}
|
||||
|
||||
// Handle new branch (oldCommitID is all zeros)
|
||||
emptyOID := objectFormat.EmptyObjectID().String()
|
||||
if oldCommitID == emptyOID || strings.Trim(oldCommitID, "0") == "" {
|
||||
// For new branches, scan all files in the new commit
|
||||
commit, err := repo.GetCommit(newCommitID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get commit %s: %w", newCommitID, err)
|
||||
}
|
||||
|
||||
// Get all entries recursively using the embedded Tree
|
||||
entries, err := commit.ListEntriesRecursiveFast()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to list tree entries: %w", err)
|
||||
}
|
||||
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
|
||||
if s.shouldIgnoreFile(entry.Name()) {
|
||||
continue
|
||||
}
|
||||
|
||||
result.ScannedFiles++
|
||||
|
||||
// Get file content
|
||||
content, err := commit.GetFileContent(entry.Name(), 1024*1024) // 1MB limit
|
||||
if err != nil {
|
||||
log.Warn("Failed to read file %s: %v", entry.Name(), err)
|
||||
continue
|
||||
}
|
||||
|
||||
secrets := s.ScanContent(content, entry.Name())
|
||||
for i := range secrets {
|
||||
secrets[i].CommitSHA = newCommitID
|
||||
}
|
||||
result.Secrets = append(result.Secrets, secrets...)
|
||||
}
|
||||
} else {
|
||||
// Get diff between commits using git diff command
|
||||
var diffBuf bytes.Buffer
|
||||
compareArg := oldCommitID + ".." + newCommitID
|
||||
err := gitcmd.NewCommand("diff", "-p", "-U0").
|
||||
AddDynamicArguments(compareArg).
|
||||
WithDir(repo.Path).
|
||||
WithStdout(&diffBuf).
|
||||
Run(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get diff: %w", err)
|
||||
}
|
||||
|
||||
// Count files changed
|
||||
filesBuf := bytes.Buffer{}
|
||||
err = gitcmd.NewCommand("diff", "--name-only").
|
||||
AddDynamicArguments(compareArg).
|
||||
WithDir(repo.Path).
|
||||
WithStdout(&filesBuf).
|
||||
Run(ctx)
|
||||
if err == nil {
|
||||
result.ScannedFiles = len(strings.Split(strings.TrimSpace(filesBuf.String()), "\n"))
|
||||
}
|
||||
|
||||
// Scan the diff
|
||||
secrets := s.ScanDiff(diffBuf.String())
|
||||
for i := range secrets {
|
||||
secrets[i].CommitSHA = newCommitID
|
||||
}
|
||||
result.Secrets = append(result.Secrets, secrets...)
|
||||
}
|
||||
|
||||
result.ScanDuration = time.Since(startTime)
|
||||
|
||||
// Determine if push should be blocked
|
||||
if setting.SecretScan.BlockOnDetection && len(result.Secrets) > 0 {
|
||||
result.Blocked = true
|
||||
result.Message = formatBlockMessage(result.Secrets)
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// maskSecret masks a secret for display
|
||||
func maskSecret(secret string) string {
|
||||
if len(secret) <= 8 {
|
||||
return strings.Repeat("*", len(secret))
|
||||
}
|
||||
|
||||
// For private keys, show type only
|
||||
if strings.Contains(secret, "-----BEGIN") {
|
||||
if idx := strings.Index(secret, "-----"); idx >= 0 {
|
||||
end := strings.Index(secret[idx+5:], "-----")
|
||||
if end > 0 {
|
||||
return fmt.Sprintf("[%s]", secret[idx+5:idx+5+end])
|
||||
}
|
||||
}
|
||||
return "[PRIVATE KEY]"
|
||||
}
|
||||
|
||||
// Show first 4 and last 4 characters
|
||||
prefix := secret[:4]
|
||||
suffix := secret[len(secret)-4:]
|
||||
masked := strings.Repeat("*", minInt(len(secret)-8, 20))
|
||||
|
||||
return prefix + masked + suffix
|
||||
}
|
||||
|
||||
// formatBlockMessage creates a user-friendly message about detected secrets
|
||||
func formatBlockMessage(secrets []DetectedSecret) string {
|
||||
var sb strings.Builder
|
||||
sb.WriteString("Push rejected: Secrets detected in your commits\n\n")
|
||||
|
||||
// Group by severity
|
||||
critical := []DetectedSecret{}
|
||||
high := []DetectedSecret{}
|
||||
other := []DetectedSecret{}
|
||||
|
||||
for _, s := range secrets {
|
||||
switch s.Severity {
|
||||
case SeverityCritical:
|
||||
critical = append(critical, s)
|
||||
case SeverityHigh:
|
||||
high = append(high, s)
|
||||
default:
|
||||
other = append(other, s)
|
||||
}
|
||||
}
|
||||
|
||||
if len(critical) > 0 {
|
||||
sb.WriteString("CRITICAL:\n")
|
||||
for _, s := range critical {
|
||||
sb.WriteString(fmt.Sprintf(" - %s in %s (line %d)\n", s.PatternName, s.FilePath, s.LineNumber))
|
||||
}
|
||||
sb.WriteString("\n")
|
||||
}
|
||||
|
||||
if len(high) > 0 {
|
||||
sb.WriteString("HIGH:\n")
|
||||
for _, s := range high {
|
||||
sb.WriteString(fmt.Sprintf(" - %s in %s (line %d)\n", s.PatternName, s.FilePath, s.LineNumber))
|
||||
}
|
||||
sb.WriteString("\n")
|
||||
}
|
||||
|
||||
if len(other) > 0 {
|
||||
sb.WriteString("OTHER:\n")
|
||||
for _, s := range other {
|
||||
sb.WriteString(fmt.Sprintf(" - %s in %s (line %d)\n", s.PatternName, s.FilePath, s.LineNumber))
|
||||
}
|
||||
sb.WriteString("\n")
|
||||
}
|
||||
|
||||
sb.WriteString("Please remove these secrets and use environment variables or a secrets manager.\n")
|
||||
sb.WriteString("If you believe this is a false positive, contact your administrator.\n")
|
||||
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
func minInt(a, b int) int {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// ScanFile scans a single file for secrets
|
||||
func (s *Scanner) ScanFile(ctx context.Context, repo *git.Repository, commitID, filePath string) ([]DetectedSecret, error) {
|
||||
if s.shouldIgnoreFile(filePath) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
commit, err := repo.GetCommit(commitID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get commit: %w", err)
|
||||
}
|
||||
|
||||
content, err := commit.GetFileContent(filePath, 1024*1024) // 1MB limit
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get file content: %w", err)
|
||||
}
|
||||
|
||||
secrets := s.ScanContent(content, filePath)
|
||||
for i := range secrets {
|
||||
secrets[i].CommitSHA = commitID
|
||||
}
|
||||
|
||||
return secrets, nil
|
||||
}
|
||||
|
||||
// ScanReader scans content from an io.Reader for secrets
|
||||
func (s *Scanner) ScanReader(r io.Reader, filePath string) ([]DetectedSecret, error) {
|
||||
if s.shouldIgnoreFile(filePath) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
content, err := io.ReadAll(io.LimitReader(r, 1024*1024)) // 1MB limit
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read content: %w", err)
|
||||
}
|
||||
|
||||
return s.ScanContent(string(content), filePath), nil
|
||||
}
|
||||
37
modules/setting/secretscan.go
Normal file
37
modules/setting/secretscan.go
Normal file
@@ -0,0 +1,37 @@
|
||||
// Copyright 2026 MarketAlly. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package setting
|
||||
|
||||
// SecretScanSettings represents the secret scanning configuration
|
||||
var SecretScan = struct {
|
||||
Enabled bool
|
||||
BlockOnDetection bool
|
||||
BlockSeverity string // "critical", "high", "medium", "low"
|
||||
ScanNewBranches bool
|
||||
IgnoredRepos []string
|
||||
IgnoredFiles []string
|
||||
AllowlistPatterns []string
|
||||
EnableVaultSuggestion bool
|
||||
}{
|
||||
Enabled: true,
|
||||
BlockOnDetection: true,
|
||||
BlockSeverity: "high", // Block on high and critical
|
||||
ScanNewBranches: true,
|
||||
IgnoredRepos: []string{},
|
||||
IgnoredFiles: []string{},
|
||||
AllowlistPatterns: []string{},
|
||||
EnableVaultSuggestion: true,
|
||||
}
|
||||
|
||||
func loadSecretScanFrom(rootCfg ConfigProvider) {
|
||||
sec := rootCfg.Section("secret_scan")
|
||||
SecretScan.Enabled = sec.Key("ENABLED").MustBool(true)
|
||||
SecretScan.BlockOnDetection = sec.Key("BLOCK_ON_DETECTION").MustBool(true)
|
||||
SecretScan.BlockSeverity = sec.Key("BLOCK_SEVERITY").MustString("high")
|
||||
SecretScan.ScanNewBranches = sec.Key("SCAN_NEW_BRANCHES").MustBool(true)
|
||||
SecretScan.IgnoredRepos = sec.Key("IGNORED_REPOS").Strings(",")
|
||||
SecretScan.IgnoredFiles = sec.Key("IGNORED_FILES").Strings(",")
|
||||
SecretScan.AllowlistPatterns = sec.Key("ALLOWLIST_PATTERNS").Strings(",")
|
||||
SecretScan.EnableVaultSuggestion = sec.Key("ENABLE_VAULT_SUGGESTION").MustBool(true)
|
||||
}
|
||||
@@ -118,6 +118,7 @@ func loadCommonSettingsFrom(cfg ConfigProvider) error {
|
||||
|
||||
loadOAuth2From(cfg)
|
||||
loadSecurityFrom(cfg)
|
||||
loadSecretScanFrom(cfg)
|
||||
if err := loadAttachmentFrom(cfg); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
issues_model "code.gitea.io/gitea/models/issues"
|
||||
perm_model "code.gitea.io/gitea/models/perm"
|
||||
access_model "code.gitea.io/gitea/models/perm/access"
|
||||
repo_model "code.gitea.io/gitea/models/repo"
|
||||
"code.gitea.io/gitea/models/unit"
|
||||
user_model "code.gitea.io/gitea/models/user"
|
||||
"code.gitea.io/gitea/modules/git"
|
||||
@@ -26,6 +27,7 @@ import (
|
||||
"code.gitea.io/gitea/services/agit"
|
||||
gitea_context "code.gitea.io/gitea/services/context"
|
||||
pull_service "code.gitea.io/gitea/services/pull"
|
||||
secretscan_service "code.gitea.io/gitea/services/secretscan"
|
||||
)
|
||||
|
||||
type preReceiveContext struct {
|
||||
@@ -151,6 +153,11 @@ func preReceiveBranch(ctx *preReceiveContext, oldCommitID, newCommitID string, r
|
||||
gitRepo := ctx.Repo.GitRepo
|
||||
objectFormat := ctx.Repo.GetObjectFormat()
|
||||
|
||||
// Secret scanning - scan push for exposed secrets
|
||||
if err := scanPushForSecrets(ctx, repo, gitRepo, oldCommitID, newCommitID); err != nil {
|
||||
return // Response already written
|
||||
}
|
||||
|
||||
if branchName == repo.DefaultBranch && newCommitID == objectFormat.EmptyObjectID().String() {
|
||||
log.Warn("Forbidden: Branch: %s is the default branch in %-v and cannot be deleted", branchName, repo)
|
||||
ctx.JSON(http.StatusForbidden, private.Response{
|
||||
@@ -538,3 +545,28 @@ func (ctx *preReceiveContext) loadPusherAndPermission() bool {
|
||||
ctx.loadedPusher = true
|
||||
return true
|
||||
}
|
||||
|
||||
// scanPushForSecrets scans the push for exposed secrets and blocks if necessary
|
||||
func scanPushForSecrets(ctx *preReceiveContext, repo *repo_model.Repository, gitRepo *git.Repository, oldCommitID, newCommitID string) error {
|
||||
result, err := secretscan_service.ScanPush(ctx, repo, gitRepo, oldCommitID, newCommitID)
|
||||
if err != nil {
|
||||
// Log the error but don't block the push on scan errors
|
||||
log.Error("Secret scan error for %s: %v", repo.FullName(), err)
|
||||
return nil
|
||||
}
|
||||
|
||||
if result.Blocked {
|
||||
log.Warn("Secret scan: Blocking push to %s - found %d secrets", repo.FullName(), len(result.Secrets))
|
||||
ctx.JSON(http.StatusForbidden, private.Response{
|
||||
UserMsg: result.Message,
|
||||
})
|
||||
return fmt.Errorf("secrets detected")
|
||||
}
|
||||
|
||||
if len(result.Secrets) > 0 {
|
||||
// Secrets found but not blocking (below threshold)
|
||||
log.Warn("Secret scan: Found %d secrets in push to %s (below blocking threshold)", len(result.Secrets), repo.FullName())
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
157
services/secretscan/secretscan.go
Normal file
157
services/secretscan/secretscan.go
Normal file
@@ -0,0 +1,157 @@
|
||||
// Copyright 2026 MarketAlly. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package secretscan
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
repo_model "code.gitea.io/gitea/models/repo"
|
||||
"code.gitea.io/gitea/modules/git"
|
||||
"code.gitea.io/gitea/modules/log"
|
||||
"code.gitea.io/gitea/modules/secretscan"
|
||||
"code.gitea.io/gitea/modules/setting"
|
||||
)
|
||||
|
||||
// ScanPushResult contains the result of scanning a push for secrets
|
||||
type ScanPushResult struct {
|
||||
Blocked bool
|
||||
Message string
|
||||
Secrets []secretscan.DetectedSecret
|
||||
RepoID int64
|
||||
RepoName string
|
||||
}
|
||||
|
||||
// ScanPush scans a push for secrets
|
||||
func ScanPush(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, oldCommitID, newCommitID string) (*ScanPushResult, error) {
|
||||
result := &ScanPushResult{
|
||||
RepoID: repo.ID,
|
||||
RepoName: repo.FullName(),
|
||||
}
|
||||
|
||||
// Check if secret scanning is enabled
|
||||
if !setting.SecretScan.Enabled {
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// Check if this repo is in the ignored list
|
||||
for _, ignored := range setting.SecretScan.IgnoredRepos {
|
||||
if strings.EqualFold(ignored, repo.FullName()) {
|
||||
log.Debug("Secret scan: Repository %s is in ignored list", repo.FullName())
|
||||
return result, nil
|
||||
}
|
||||
}
|
||||
|
||||
// Create scanner
|
||||
scanner := secretscan.NewScanner()
|
||||
|
||||
// Scan the commit range
|
||||
scanResult, err := scanner.ScanCommitRange(ctx, gitRepo, oldCommitID, newCommitID)
|
||||
if err != nil {
|
||||
log.Error("Secret scan failed for %s: %v", repo.FullName(), err)
|
||||
// Don't block on scan errors - just log and continue
|
||||
return result, nil
|
||||
}
|
||||
|
||||
if len(scanResult.Secrets) == 0 {
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// Filter by severity
|
||||
filteredSecrets := filterBySeverity(scanResult.Secrets, setting.SecretScan.BlockSeverity)
|
||||
|
||||
if len(filteredSecrets) == 0 {
|
||||
// Secrets found but below blocking threshold
|
||||
log.Warn("Secret scan: Found %d secrets in %s (below threshold)", len(scanResult.Secrets), repo.FullName())
|
||||
result.Secrets = scanResult.Secrets
|
||||
return result, nil
|
||||
}
|
||||
|
||||
result.Secrets = filteredSecrets
|
||||
result.Blocked = setting.SecretScan.BlockOnDetection
|
||||
result.Message = formatBlockMessage(filteredSecrets, repo.FullName())
|
||||
|
||||
log.Warn("Secret scan: Blocking push to %s - found %d secrets", repo.FullName(), len(filteredSecrets))
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// filterBySeverity filters secrets based on the minimum severity level
|
||||
func filterBySeverity(secrets []secretscan.DetectedSecret, minSeverity string) []secretscan.DetectedSecret {
|
||||
severityLevel := map[secretscan.Severity]int{
|
||||
secretscan.SeverityCritical: 4,
|
||||
secretscan.SeverityHigh: 3,
|
||||
secretscan.SeverityMedium: 2,
|
||||
secretscan.SeverityLow: 1,
|
||||
}
|
||||
|
||||
minLevel := severityLevel[secretscan.Severity(minSeverity)]
|
||||
if minLevel == 0 {
|
||||
minLevel = severityLevel[secretscan.SeverityHigh] // Default to high
|
||||
}
|
||||
|
||||
var filtered []secretscan.DetectedSecret
|
||||
for _, s := range secrets {
|
||||
if severityLevel[s.Severity] >= minLevel {
|
||||
filtered = append(filtered, s)
|
||||
}
|
||||
}
|
||||
|
||||
return filtered
|
||||
}
|
||||
|
||||
// formatBlockMessage creates a user-friendly error message
|
||||
func formatBlockMessage(secrets []secretscan.DetectedSecret, repoName string) string {
|
||||
var sb strings.Builder
|
||||
|
||||
sb.WriteString("═══════════════════════════════════════════════════════════\n")
|
||||
sb.WriteString(" 🔒 SECRET DETECTION - PUSH BLOCKED\n")
|
||||
sb.WriteString("═══════════════════════════════════════════════════════════\n\n")
|
||||
|
||||
sb.WriteString(fmt.Sprintf("Repository: %s\n", repoName))
|
||||
sb.WriteString(fmt.Sprintf("Secrets found: %d\n\n", len(secrets)))
|
||||
|
||||
// Group by file
|
||||
byFile := make(map[string][]secretscan.DetectedSecret)
|
||||
for _, s := range secrets {
|
||||
byFile[s.FilePath] = append(byFile[s.FilePath], s)
|
||||
}
|
||||
|
||||
for file, fileSecrets := range byFile {
|
||||
sb.WriteString(fmt.Sprintf("📄 %s\n", file))
|
||||
for _, s := range fileSecrets {
|
||||
icon := "⚠️"
|
||||
if s.Severity == secretscan.SeverityCritical {
|
||||
icon = "🔴"
|
||||
} else if s.Severity == secretscan.SeverityHigh {
|
||||
icon = "🟠"
|
||||
}
|
||||
sb.WriteString(fmt.Sprintf(" %s Line %d: %s [%s]\n", icon, s.LineNumber, s.PatternName, s.Severity))
|
||||
sb.WriteString(fmt.Sprintf(" Found: %s\n", s.MaskedText))
|
||||
}
|
||||
sb.WriteString("\n")
|
||||
}
|
||||
|
||||
sb.WriteString("───────────────────────────────────────────────────────────\n")
|
||||
sb.WriteString("HOW TO FIX:\n")
|
||||
sb.WriteString(" 1. Remove the secrets from your code\n")
|
||||
sb.WriteString(" 2. Use environment variables or a secrets manager\n")
|
||||
sb.WriteString(" 3. Consider using GitCaddy Vault to securely store secrets\n")
|
||||
sb.WriteString("\n")
|
||||
sb.WriteString("If this is a false positive, contact your administrator.\n")
|
||||
sb.WriteString("═══════════════════════════════════════════════════════════\n")
|
||||
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
// IsEnabled returns whether secret scanning is enabled
|
||||
func IsEnabled() bool {
|
||||
return setting.SecretScan.Enabled
|
||||
}
|
||||
|
||||
// ShouldBlockOnDetection returns whether to block pushes with secrets
|
||||
func ShouldBlockOnDetection() bool {
|
||||
return setting.SecretScan.BlockOnDetection
|
||||
}
|
||||
Reference in New Issue
Block a user