feat(vault): add database migration system for vault plugin

Implements explicit column migration logic to handle schema upgrades that xorm.Sync() doesn't reliably perform. Adds encryption_mode column migration for vault_secret table to support lockbox (E2E) encryption. Includes database-agnostic column existence checks and ALTER TABLE statement generation for PostgreSQL, MySQL, SQLite, and MSSQL.

.notes/note-1768964774362-3y9dna8sm.json

@@ -1,7 +1,7 @@
 {
   "id": "note-1768964774362-3y9dna8sm",
   "title": "working",
-  "content": "● Done. Updated all vault locale files with the new translation keys.\n\n  Summary of what's now in gitcaddy-vault:\n  1. Templates: list.tmpl, view.tmpl, audit.tmpl, tokens.tmpl (copied)\n  2. Locale files: Added 20+ new translation keys to all locale files\n\n  What stays in gitcaddy-server:\n  - Router handlers (routers/web/repo/vault/vault.go) - these call the vault service\n  - Service interface definitions (services/vault/vault.go)\n  - Locale files also need the keys (for ctx.Locale.Tr calls in templates)\n\n```\n  Now you need to:\n  1. Commit and tag vault as v1.0.32\n  2. Update server go.mod to v1.0.32\n  3. Release server v3.1.10\n  4. Deploy to git-test\n```\n\n● So:\n\n  | Key                       | Value                                                            |\n  |---------------------------|------------------------------------------------------------------|\n  | Old (SECRET_KEY fallback) | bfe6e73282adba6545c2c9ba3542a332a80d62d1644b3b24c14b020a838c136f |\n  | New (MASTER_KEY)          | af1a7e9d7fe73258800b434ab5ffe7bf1ee3dff49740ef3cc3c5c014587acc08 |\n",
+  "content": "● Done. Updated all vault locale files with the new translation keys.\n\n  Summary of what's now in gitcaddy-vault:\n  1. Templates: list.tmpl, view.tmpl, audit.tmpl, tokens.tmpl (copied)\n  2. Locale files: Added 20+ new translation keys to all locale files\n\n  What stays in gitcaddy-server:\n  - Router handlers (routers/web/repo/vault/vault.go) - these call the vault service\n  - Service interface definitions (services/vault/vault.go)\n  - Locale files also need the keys (for ctx.Locale.Tr calls in templates)\n\n```\n  Now you need to:\n  1. Commit and tag vault as v1.0.32\n  2. Update server go.mod to v1.0.32\n  3. Release server v3.1.10\n  4. Deploy to git-test\n```\n",
   "createdAt": 1768964774360,
-  "updatedAt": 1770432337981
+  "updatedAt": 1770512254003
 }

README.md

@@ -23,6 +23,30 @@ GitCaddy Vault is a commercial module compiled directly into GitCaddy Server tha
 | `certificate` | TLS/SSL certificates |
 | `ssh-key` | SSH private keys |
 
+### Lockbox (End-to-End Encryption)
+
+Lockbox provides optional client-side encryption where the server **never sees your plaintext secrets**. Perfect for highly sensitive data where you don't want even the server administrator to have access.
+
+| Feature | Standard Mode | Lockbox Mode |
+|---------|--------------|--------------|
+| Server sees plaintext | Yes | No |
+| Passphrase required | No | Yes |
+| Recovery if passphrase lost | Yes | No |
+| Web UI viewing | Yes | CLI/SDK only |
+
+**How Lockbox Works:**
+1. Client encrypts secret with your passphrase using Argon2id + AES-256-GCM
+2. Encrypted blob is sent to server in `lockbox:v1:...` format
+3. Server wraps the blob with repository DEK (double encryption)
+4. On retrieval, server unwraps DEK layer, returns lockbox blob
+5. Client decrypts with your passphrase
+
+**Encryption Scheme:**
+- Key derivation: Argon2id (time=1, memory=64MB, parallelism=4)
+- Cipher: AES-256-GCM with 12-byte nonce
+- Salt: 16 bytes random per secret
+- Format: `lockbox:v1:<base64(salt)>:<base64(nonce||ciphertext||tag)>`
+
 ### Security Architecture
 
 ```

models/secret.go

@@ -37,6 +37,9 @@ type VaultSecret struct {
 	CreatedBy      int64              `xorm:"NOT NULL"`
 	UpdatedUnix    timeutil.TimeStamp `xorm:"updated NOT NULL"`
 
+	// Encryption mode: "standard" (server-side) or "lockbox" (client E2E encrypted)
+	EncryptionMode string `xorm:"VARCHAR(20) NOT NULL DEFAULT 'standard'"`
+
 	// Soft delete
 	DeletedUnix timeutil.TimeStamp `xorm:"INDEX"` // 0 = active, >0 = soft deleted
 	DeletedBy   int64

plugin.go

@@ -5,6 +5,7 @@ package vault
 
 import (
 	"context"
+	"fmt"
 
 	"git.marketally.com/gitcaddy/gitcaddy-vault/crypto"
 	"git.marketally.com/gitcaddy/gitcaddy-vault/license"
@@ -98,7 +99,99 @@ func (p *VaultPlugin) RegisterModels() []any {
 
 // Migrate runs database migrations for this plugin
 func (p *VaultPlugin) Migrate(ctx context.Context, x *xorm.Engine) error {
-	return x.Sync(p.RegisterModels()...)
+	// First, sync tables (works for fresh installations)
+	if err := x.Sync(p.RegisterModels()...); err != nil {
+		return err
+	}
+
+	// Then run explicit column migrations for upgrades
+	// xorm.Sync() doesn't reliably add columns to existing tables
+	return runColumnMigrations(ctx, x)
+}
+
+// runColumnMigrations adds any missing columns to existing tables.
+// This handles the case where xorm.Sync() fails to add new columns.
+func runColumnMigrations(ctx context.Context, x *xorm.Engine) error {
+	migrations := []struct {
+		table      string
+		column     string
+		columnType string
+		defaultVal string
+	}{
+		// v1.0.31: Add encryption_mode column for lockbox (E2E) secrets
+		{"vault_secret", "encryption_mode", "VARCHAR(20)", "'standard'"},
+	}
+
+	for _, m := range migrations {
+		exists, err := columnExists(x, m.table, m.column)
+		if err != nil {
+			return err
+		}
+		if !exists {
+			log.Info("Vault migration: adding column %s.%s", m.table, m.column)
+			sql := buildAddColumnSQL(x, m.table, m.column, m.columnType, m.defaultVal)
+			if _, err := x.Exec(sql); err != nil {
+				return fmt.Errorf("failed to add column %s.%s: %w", m.table, m.column, err)
+			}
+		}
+	}
+	return nil
+}
+
+// columnExists checks if a column exists in a table
+func columnExists(x *xorm.Engine, table, column string) (bool, error) {
+	dialect := x.Dialect().URI().DBType
+	var sql string
+
+	switch dialect {
+	case "postgres":
+		sql = fmt.Sprintf(`SELECT 1 FROM information_schema.columns WHERE table_name = '%s' AND column_name = '%s'`, table, column)
+	case "mysql":
+		sql = fmt.Sprintf(`SELECT 1 FROM information_schema.columns WHERE table_name = '%s' AND column_name = '%s'`, table, column)
+	case "sqlite3":
+		// SQLite uses PRAGMA
+		results, err := x.Query(fmt.Sprintf("PRAGMA table_info(%s)", table))
+		if err != nil {
+			return false, err
+		}
+		for _, row := range results {
+			if string(row["name"]) == column {
+				return true, nil
+			}
+		}
+		return false, nil
+	case "mssql":
+		sql = fmt.Sprintf(`SELECT 1 FROM sys.columns WHERE object_id = OBJECT_ID('%s') AND name = '%s'`, table, column)
+	default:
+		// Fallback: try postgres-style
+		sql = fmt.Sprintf(`SELECT 1 FROM information_schema.columns WHERE table_name = '%s' AND column_name = '%s'`, table, column)
+	}
+
+	results, err := x.Query(sql)
+	if err != nil {
+		return false, err
+	}
+	return len(results) > 0, nil
+}
+
+// buildAddColumnSQL builds the appropriate ALTER TABLE statement for the database type
+func buildAddColumnSQL(x *xorm.Engine, table, column, columnType, defaultVal string) string {
+	dialect := x.Dialect().URI().DBType
+
+	switch dialect {
+	case "postgres":
+		return fmt.Sprintf(`ALTER TABLE "%s" ADD COLUMN IF NOT EXISTS "%s" %s NOT NULL DEFAULT %s`, table, column, columnType, defaultVal)
+	case "mysql":
+		// MySQL doesn't have IF NOT EXISTS for columns, but we already checked
+		return fmt.Sprintf("ALTER TABLE `%s` ADD COLUMN `%s` %s NOT NULL DEFAULT %s", table, column, columnType, defaultVal)
+	case "sqlite3":
+		return fmt.Sprintf(`ALTER TABLE "%s" ADD COLUMN "%s" %s NOT NULL DEFAULT %s`, table, column, columnType, defaultVal)
+	case "mssql":
+		return fmt.Sprintf(`ALTER TABLE [%s] ADD [%s] %s NOT NULL DEFAULT %s`, table, column, columnType, defaultVal)
+	default:
+		// Fallback to standard SQL
+		return fmt.Sprintf(`ALTER TABLE "%s" ADD COLUMN "%s" %s NOT NULL DEFAULT %s`, table, column, columnType, defaultVal)
+	}
 }
 
 // RegisterRepoWebRoutes adds vault routes under /{owner}/{repo}/vault

routes/routes.go

@@ -49,6 +49,7 @@ type SecretResponse struct {
 	Name           string `json:"name"`
 	Description    string `json:"description,omitempty"`
 	Type           string `json:"type"`
+	EncryptionMode string `json:"encryption_mode,omitempty"` // "standard" or "lockbox"
 	CurrentVersion int    `json:"current_version"`
 	CreatedAt      int64  `json:"created_at"`
 	UpdatedAt      int64  `json:"updated_at"`
@@ -115,11 +116,12 @@ type TokenInfoResponse struct {
 
 // CreateSecretRequest is the request body for creating a secret
 type CreateSecretRequest struct {
-	Name        string `json:"name" binding:"required"`
-	Description string `json:"description"`
-	Type        string `json:"type"`
-	Value       string `json:"value" binding:"required"`
-	Comment     string `json:"comment"` // Used when updating existing secret
+	Name           string `json:"name" binding:"required"`
+	Description    string `json:"description"`
+	Type           string `json:"type"`
+	Value          string `json:"value" binding:"required"`
+	Comment        string `json:"comment"`        // Used when updating existing secret
+	EncryptionMode string `json:"encryption_mode"` // "standard" (default) or "lockbox"
 }
 
 // UpdateSecretRequest is the request body for updating a secret
@@ -272,10 +274,15 @@ func requireRepoWrite(ctx *context.APIContext) bool {
 }
 
 func secretToResponse(s *models.VaultSecret) SecretResponse {
+	encMode := s.EncryptionMode
+	if encMode == "" {
+		encMode = "standard"
+	}
 	return SecretResponse{
 		Name:           s.Name,
 		Description:    s.Description,
 		Type:           string(s.Type),
+		EncryptionMode: encMode,
 		CurrentVersion: s.CurrentVersion,
 		CreatedAt:      int64(s.CreatedUnix),
 		UpdatedAt:      int64(s.UpdatedUnix),
@@ -564,6 +571,28 @@ func apiPutSecret(lic *license.Manager) http.HandlerFunc {
 			return
 		}
 
+		// Validate encryption mode
+		encryptionMode := req.EncryptionMode
+		if encryptionMode == "" {
+			encryptionMode = "standard"
+		}
+		if encryptionMode != "standard" && encryptionMode != "lockbox" {
+			ctx.JSON(http.StatusBadRequest, map[string]any{
+				"error":   "invalid_encryption_mode",
+				"message": "encryption_mode must be 'standard' or 'lockbox'",
+			})
+			return
+		}
+
+		// Validate lockbox format: must start with "lockbox:v1:"
+		if encryptionMode == "lockbox" && !strings.HasPrefix(req.Value, "lockbox:v1:") {
+			ctx.JSON(http.StatusBadRequest, map[string]any{
+				"error":   "invalid_lockbox_format",
+				"message": "Lockbox secrets must be pre-encrypted with format 'lockbox:v1:<salt>:<ciphertext>'",
+			})
+			return
+		}
+
 		// Check if secret exists
 		existing, err := services.GetSecret(ctx, ctx.Repo.Repository.ID, name)
 		if err == services.ErrSecretNotFound {
@@ -576,11 +605,12 @@ func apiPutSecret(lic *license.Manager) http.HandlerFunc {
 				limits = &info.Limits
 			}
 			secret, err := services.CreateSecret(ctx, ctx.Repo.Repository.ID, services.CreateSecretOptions{
-				Name:        req.Name,
-				Description: req.Description,
-				Type:        req.Type,
-				Value:       req.Value,
-				CreatorID:   ctx.Doer.ID,
+				Name:           req.Name,
+				Description:    req.Description,
+				Type:           req.Type,
+				Value:          req.Value,
+				EncryptionMode: encryptionMode,
+				CreatorID:      ctx.Doer.ID,
 			}, limits)
 			if err != nil {
 				switch err {

services/secrets.go

@@ -106,11 +106,12 @@ func GetSecretValue(ctx context.Context, repoID int64, name string, version int)
 
 // CreateSecretOptions contains options for creating a secret
 type CreateSecretOptions struct {
-	Name        string
-	Description string
-	Type        string
-	Value       string
-	CreatorID   int64
+	Name           string
+	Description    string
+	Type           string
+	Value          string
+	EncryptionMode string // "standard" (default) or "lockbox"
+	CreatorID      int64
 }
 
 // CreateSecret creates a new secret
@@ -158,12 +159,19 @@ func CreateSecret(ctx context.Context, repoID int64, opts CreateSecretOptions, l
 
 	now := timeutil.TimeStampNow()
 
+	// Determine encryption mode
+	encryptionMode := opts.EncryptionMode
+	if encryptionMode == "" {
+		encryptionMode = "standard"
+	}
+
 	// Create the secret
 	secret := &models.VaultSecret{
 		RepoID:         repoID,
 		Name:           opts.Name,
 		Description:    opts.Description,
 		Type:           models.SecretType(opts.Type),
+		EncryptionMode: encryptionMode,
 		CurrentVersion: 1,
 		CreatedUnix:    now,
 		UpdatedUnix:    now,

templates/repo/vault/key_migrate.tmpl

@@ -45,7 +45,7 @@
 		<div class="ui info message">
 			<p>{{ctx.Locale.Tr "vault.migrate_from_fallback_description"}}</p>
 		</div>
-		<form class="ui form" id="migrate-fallback-form" action="{{.RepoLink}}/vault/migrate-from-fallback" method="post">
+		<form class="ui form ignore-dirty" id="migrate-fallback-form" action="{{.RepoLink}}/vault/migrate-from-fallback" method="post">
 			{{.CsrfTokenHtml}}
 			<div class="field">
 				<select name="scope" class="ui dropdown">
@@ -86,7 +86,7 @@
 		<div class="ui divider"></div>
 		{{end}}
 
-		<form class="ui form" id="migrate-manual-form" action="{{.RepoLink}}/vault/migrate-key" method="post">
+		<form class="ui form ignore-dirty" id="migrate-manual-form" action="{{.RepoLink}}/vault/migrate-key" method="post">
 			{{.CsrfTokenHtml}}
 			<div class="required field">
 				<label>{{ctx.Locale.Tr "vault.old_master_key"}}</label>
@@ -119,7 +119,7 @@
 		</h5>
 		<p>{{ctx.Locale.Tr "vault.dek_rotation_description"}}</p>
 		{{if .IsEnterprise}}
-		<form class="ui form" id="rotate-dek-form" action="{{.RepoLink}}/vault/rotate-key" method="post">
+		<form class="ui form ignore-dirty" id="rotate-dek-form" action="{{.RepoLink}}/vault/rotate-key" method="post">
 			{{.CsrfTokenHtml}}
 			<button class="ui button show-modal" data-modal="#rotate-dek-modal" type="button">
 				{{svg "octicon-sync" 16}} {{ctx.Locale.Tr "vault.rotate_dek"}}

templates/repo/vault/list.tmpl

@@ -65,8 +65,11 @@
 							<tr{{if .IsDeleted}} class="negative"{{end}}>
 									<td>
 											<a href="{{$.RepoLink}}/vault/secrets/{{.Name}}">
-													{{svg "octicon-key" 16}} <strong>{{.Name}}</strong>
+													{{if eq .EncryptionMode "lockbox"}}{{svg "octicon-lock" 16}}{{else}}{{svg "octicon-key" 16}}{{end}} <strong>{{.Name}}</strong>
 											</a>
+											{{if eq .EncryptionMode "lockbox"}}
+											<span class="ui tiny blue label" data-tooltip="End-to-end encrypted">{{svg "octicon-shield-lock" 10}} Lock-Box</span>
+											{{end}}
 											{{if .Description}}
 											<br><small class="text grey">{{.Description}}</small>
 											{{end}}

templates/repo/vault/view.tmpl

@@ -3,7 +3,12 @@
 		<div class="tw-flex tw-items-start tw-justify-between tw-gap-4">
 			<div>
 				<h4 class="ui header tw-mb-0">
+					{{if eq .Secret.EncryptionMode "lockbox"}}
+					{{svg "octicon-lock" 20}} {{.Secret.Name}}
+					<span class="ui blue label" data-tooltip="End-to-end encrypted. Use CLI/SDK to decrypt.">{{svg "octicon-shield-lock" 12}} Lock-Box</span>
+					{{else}}
 					{{svg "octicon-key" 20}} {{.Secret.Name}}
+					{{end}}
 					{{if .Secret.IsDeleted}}
 					<span class="ui red label">{{ctx.Locale.Tr "vault.deleted"}}</span>
 					{{end}}
@@ -50,7 +55,52 @@
 		</div>
 	</div>
 
-	{{if .CanWrite}}
+	{{if eq .Secret.EncryptionMode "lockbox"}}
+	<!-- Lockbox secret - show locked state -->
+	<div class="ui segment tw-text-center">
+		<div class="tw-py-8">
+			{{svg "octicon-lock" 64 "tw-text-blue-500"}}
+			<h3 class="ui header tw-mt-4">Lock-Box Protected</h3>
+			<p class="tw-text-secondary tw-max-w-md tw-mx-auto">
+				This secret is end-to-end encrypted. The server cannot decrypt it.
+				Use the CLI or SDK with your passphrase to access the value.
+			</p>
+			<div class="ui divider"></div>
+			<div class="ui secondary segment tw-text-left tw-max-w-lg tw-mx-auto">
+				<p class="tw-font-semibold tw-mb-2">CLI Usage:</p>
+				<code class="tw-block tw-p-2 tw-bg-gray-100 dark:tw-bg-gray-800 tw-rounded tw-font-mono tw-text-sm">
+					vault-resolve get --lockbox {{.Secret.Name}}
+				</code>
+				<p class="tw-font-semibold tw-mb-2 tw-mt-4">Go SDK:</p>
+				<code class="tw-block tw-p-2 tw-bg-gray-100 dark:tw-bg-gray-800 tw-rounded tw-font-mono tw-text-sm">
+					value, err := client.GetLockbox(ctx, "{{.Secret.Name}}", passphrase)
+				</code>
+			</div>
+			{{if .CanWrite}}
+			<div class="tw-mt-4">
+				{{if .Secret.IsDeleted}}
+				<button class="ui green button" type="button" onclick="document.getElementById('restore-form').submit();">
+					{{svg "octicon-history" 16}} {{ctx.Locale.Tr "vault.restore"}}
+				</button>
+				{{else}}
+				<button class="ui red button show-modal" type="button" data-modal="#delete-secret-modal">
+					{{svg "octicon-trash" 16}} {{ctx.Locale.Tr "vault.delete"}}
+				</button>
+				{{end}}
+			</div>
+			{{if .Secret.IsDeleted}}
+			<form id="restore-form" class="tw-hidden" action="{{.RepoLink}}/vault/secrets/{{.Secret.Name}}/restore" method="post">
+				{{.CsrfTokenHtml}}
+			</form>
+			{{else}}
+			<form id="delete-form" class="tw-hidden" action="{{.RepoLink}}/vault/secrets/{{.Secret.Name}}/delete" method="post">
+				{{.CsrfTokenHtml}}
+			</form>
+			{{end}}
+			{{end}}
+		</div>
+	</div>
+	{{else if .CanWrite}}
 	<!-- Editor view for users who can write -->
 	<div class="ui segment">
 		<h5 class="ui header">{{ctx.Locale.Tr "vault.edit_secret"}}</h5>
@@ -125,8 +175,8 @@
 		});
 	})();
 	</script>
-	{{else}}
-	<!-- Read-only view for users who cannot write -->
+	{{else if ne .Secret.EncryptionMode "lockbox"}}
+	<!-- Read-only view for users who cannot write (standard secrets only) -->
 	<div class="ui segment">
 		<div class="tw-flex tw-items-center tw-justify-between tw-mb-4">
 			<h5 class="ui header tw-mb-0">{{ctx.Locale.Tr "vault.secret_value"}}</h5>