package backup import ( "archive/zip" "bufio" "context" "crypto/rand" "encoding/json" "fmt" "io" "log" "os" "os/exec" "path/filepath" "strconv" "strings" "time" ) // KopiaStrategy implements the backup strategy using Kopia type KopiaStrategy struct { Retention Retention Provider KopiaProvider ConfigPath string } // NewKopiaStrategy creates a new kopia strategy with specified provider func NewKopiaStrategy(retention Retention, provider KopiaProvider, configPath string) *KopiaStrategy { return &KopiaStrategy{ Retention: retention, Provider: provider, ConfigPath: configPath, } } // Execute performs the kopia backup func (s *KopiaStrategy) Execute(ctx context.Context, serviceName, directory string) error { log.Printf("Performing kopia backup for service %s: %s", serviceName, directory) // Get or create password for this service password, err := getOrCreatePassword(serviceName, 48) if err != nil { return fmt.Errorf("failed to get or create password: %w", err) } // Ensure Kopia config directory exists kopiaConfigDir := filepath.Join(os.Getenv("HOME"), ".kopia") if err := os.MkdirAll(kopiaConfigDir, 0755); err != nil { return fmt.Errorf("failed to create kopia config directory: %w", err) } // Repository not connected, connect or create via provider log.Printf("Connecting to repository for %s", serviceName) if err := s.Provider.Connect(ctx, serviceName, password, s.ConfigPath); err != nil { return fmt.Errorf("failed to connect to repository: %w", err) } // Create snapshot log.Printf("Creating snapshot for directory: %s", directory) snapshotCmd := exec.Command("kopia", "--config-file", s.ConfigPath, "snapshot", "create", directory) snapshotOutput, err := snapshotCmd.CombinedOutput() if err != nil { return fmt.Errorf("failed to create snapshot: %w\nOutput: %s", err, snapshotOutput) } // Set retention policy log.Printf("Setting retention policy for %s", serviceName) args := []string{ "--config-file", s.ConfigPath, "policy", "set", "--keep-latest", fmt.Sprintf("%d", s.Retention.KeepLatest), "--keep-hourly", fmt.Sprintf("%d", s.Retention.KeepHourly), "--keep-daily", fmt.Sprintf("%d", s.Retention.KeepDaily), "--keep-weekly", fmt.Sprintf("%d", s.Retention.KeepWeekly), "--keep-monthly", fmt.Sprintf("%d", s.Retention.KeepMonthly), "--keep-annual", fmt.Sprintf("%d", s.Retention.KeepYearly), directory, } policyCmd := exec.Command("kopia", args...) policyOutput, err := policyCmd.CombinedOutput() if err != nil { return fmt.Errorf("failed to set policy: %w\nOutput: %s", err, policyOutput) } log.Printf("Snapshot and policy set successfully for %s", serviceName) return nil } // ListBackups returns information about existing backups func (s *KopiaStrategy) ListBackups(ctx context.Context, serviceName string) ([]BackupInfo, error) { // Parse service group and index from service name (e.g., "backealocal.1") groupName, _, err := parseServiceName(serviceName) if err != nil { return nil, fmt.Errorf("invalid service name format: %w", err) } // Get service directory from config factory, err := NewBackupFactory("config.yml") if err != nil { return nil, fmt.Errorf("failed to create factory: %w", err) } // Find service group serviceGroup, exists := factory.Config.Services[groupName] if !exists { return nil, fmt.Errorf("service group not found: %s", groupName) } // Get directory from the service group level directoryPath := strings.TrimRight(serviceGroup.Source.Path, "/") // Ensure we're connected to the repository err = s.EnsureRepositoryConnected(ctx, serviceName) if err != nil { return nil, fmt.Errorf("failed to connect to repository: %w", err) } // Run kopia snapshot list command with JSON output cmd := exec.CommandContext( ctx, "kopia", "--config-file", s.ConfigPath, "snapshot", "list", "--json", ) output, err := cmd.Output() if err != nil { return nil, fmt.Errorf("failed to list snapshots: %w", err) } // Parse the JSON output var snapshots []map[string]interface{} if err := json.Unmarshal(output, &snapshots); err != nil { return nil, fmt.Errorf("failed to parse snapshot list: %w", err) } // Convert to BackupInfo var result []BackupInfo for _, snap := range snapshots { // Get basic info id, _ := snap["id"].(string) endTime, _ := snap["endTime"].(string) // Only include snapshots for the requested service by directory path source, ok := snap["source"].(map[string]interface{}) if !ok { continue } sourcePath, _ := source["path"].(string) // Match by exact directory path, not service name if sourcePath != directoryPath { continue } // Parse time var creationTime time.Time if endTime != "" { t, err := time.Parse(time.RFC3339, endTime) if err == nil { creationTime = t } } // Get size information from stats var size int64 if stats, ok := snap["stats"].(map[string]interface{}); ok { if totalSize, ok := stats["totalSize"].(float64); ok { size = int64(totalSize) } } // Determine retention tag retentionTag := "none" if reasons, ok := snap["retentionReason"].([]interface{}); ok && len(reasons) > 0 { // Get the first reason which indicates the highest priority retention if reason, ok := reasons[0].(string); ok { parts := strings.SplitN(reason, "-", 2) if len(parts) > 0 { retentionTag = parts[0] } } } result = append(result, BackupInfo{ ID: id, CreationTime: creationTime, Size: size, Source: sourcePath, Type: "kopia", RetentionTag: retentionTag, }) } return result, nil } // GetStorageUsage returns information about the total storage used by the repository func (s *KopiaStrategy) GetStorageUsage(ctx context.Context, serviceName string) (*StorageUsageInfo, error) { // Ensure we're connected to the repository err := s.EnsureRepositoryConnected(ctx, serviceName) if err != nil { return nil, fmt.Errorf("failed to connect to repository: %w", err) } // Get provider type (b2, local, sftp) providerType := "unknown" switch s.Provider.(type) { case *KopiaB2Provider: providerType = "b2" case *KopiaLocalProvider: providerType = "local" case *KopiaSFTPProvider: providerType = "sftp" } // Initialize storage info info := &StorageUsageInfo{ Provider: providerType, ProviderID: s.Provider.GetBucketName(serviceName), } // Calculate logical size by summing up the sizes of all snapshots backups, err := s.ListBackups(ctx, serviceName) if err == nil { var totalLogicalBytes int64 for _, backup := range backups { totalLogicalBytes += backup.Size } info.TotalBytes = totalLogicalBytes } // Try to get physical storage stats using blob stats command blobStatsCmd := exec.CommandContext( ctx, "kopia", "--config-file", s.ConfigPath, "blob", "stats", ) blobStatsOutput, err := blobStatsCmd.CombinedOutput() if err == nil { // Parse the text output outputStr := string(blobStatsOutput) log.Printf("Blob stats output: %s", outputStr) // Look for the line with "Total:" lines := strings.Split(outputStr, "\n") for _, line := range lines { line = strings.TrimSpace(line) if strings.HasPrefix(line, "Total:") { parts := strings.SplitN(line, ":", 2) if len(parts) == 2 { sizeStr := strings.TrimSpace(parts[1]) size, err := parseHumanSize(sizeStr) if err == nil { info.TotalBytes = size log.Printf("Got physical size from blob stats: %d bytes", size) break } else { log.Printf("Failed to parse size '%s': %v", sizeStr, err) } } } } } else { log.Printf("Blob stats command failed: %v - %s", err, string(blobStatsOutput)) } return info, nil } // Helper method to ensure the repository is connected before operations func (s *KopiaStrategy) EnsureRepositoryConnected(ctx context.Context, serviceName string) error { // Check if kopia repository is already connected with config file cmd := exec.Command("kopia", "--config-file", s.ConfigPath, "repository", "status") err := cmd.Run() if err != nil { // Repository not connected, try to connect password, err := getOrCreatePassword(serviceName, 48) if err != nil { return fmt.Errorf("failed to get password: %w", err) } // Connect using provider if err := s.Provider.Connect(ctx, serviceName, password, s.ConfigPath); err != nil { return fmt.Errorf("failed to connect to repository: %w", err) } } return nil } // parseServiceName splits a service name in the format "group.index" into its components func parseServiceName(serviceName string) (string, string, error) { parts := strings.SplitN(serviceName, ".", 2) if len(parts) != 2 { return "", "", fmt.Errorf("service name must be in format 'group.index', got '%s'", serviceName) } return parts[0], parts[1], nil } // getOrCreatePassword retrieves a password from kopia.env or creates a new one if it doesn't exist // Extracted to a package-level function since it's utility functionality func getOrCreatePassword(serviceName string, length int) (string, error) { // Define the expected key in the env file // Replace dots with underscores for environment variable name safeServiceName := strings.ReplaceAll(serviceName, ".", "_") passwordKey := fmt.Sprintf("KOPIA_%s_PASSWORD", strings.ToUpper(safeServiceName)) // Try to read from kopia.env first kopiaEnvPath := "kopia.env" if _, err := os.Stat(kopiaEnvPath); err == nil { // File exists, check if the password is already there file, err := os.Open(kopiaEnvPath) if err != nil { return "", fmt.Errorf("failed to open kopia.env: %w", err) } defer file.Close() scanner := bufio.NewScanner(file) for scanner.Scan() { line := scanner.Text() if strings.HasPrefix(line, passwordKey+"=") { parts := strings.SplitN(line, "=", 2) if len(parts) == 2 { // Found the password, remove quotes if present password := parts[1] password = strings.Trim(password, "\"") return password, nil } } } if err := scanner.Err(); err != nil { return "", fmt.Errorf("error reading kopia.env: %w", err) } } // Password not found or file doesn't exist, generate a new password password, err := generateSecurePassword(length) if err != nil { return "", fmt.Errorf("failed to generate password: %w", err) } // Create or append to kopia.env file, err := os.OpenFile(kopiaEnvPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600) if err != nil { return "", fmt.Errorf("failed to open kopia.env for writing: %w", err) } defer file.Close() // Write the new password entry with quotes passwordEntry := fmt.Sprintf("%s=\"%s\"\n", passwordKey, password) if _, err := file.WriteString(passwordEntry); err != nil { return "", fmt.Errorf("failed to write to kopia.env: %w", err) } log.Printf("Created new password for service %s and stored in kopia.env", serviceName) return password, nil } // generateSecurePassword creates a cryptographically secure random password // using only safe characters that work well with command line tools func generateSecurePassword(length int) (string, error) { // Use a more robust but safe character set // Avoiding characters that might cause shell interpretation issues // No quotes, backslashes, spaces, or common special chars that need escaping const chars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" result := make([]byte, length) // Create a secure source of randomness randomBytes := make([]byte, length) if _, err := rand.Read(randomBytes); err != nil { return "", err } // Ensure minimum complexity requirements // At least 2 uppercase, 2 lowercase, 2 digits if length >= 6 { // Force first characters to meet minimum requirements result[0] = 'A' + byte(randomBytes[0]%26) // Uppercase result[1] = 'A' + byte(randomBytes[1]%26) // Uppercase result[2] = 'a' + byte(randomBytes[2]%26) // Lowercase result[3] = 'a' + byte(randomBytes[3]%26) // Lowercase result[4] = '0' + byte(randomBytes[4]%10) // Digit result[5] = '0' + byte(randomBytes[5]%10) // Digit // Fill the rest with random chars for i := 6; i < length; i++ { result[i] = chars[int(randomBytes[i])%len(chars)] } // Shuffle the result to avoid predictable pattern for i := length - 1; i > 0; i-- { j := int(randomBytes[i]) % (i + 1) result[i], result[j] = result[j], result[i] } } else { // For very short passwords just use random chars for i := 0; i < length; i++ { result[i] = chars[int(randomBytes[i])%len(chars)] } } return string(result), nil } // parseHumanSize parses a human-readable size string (e.g., "32.8 MB") into bytes func parseHumanSize(sizeStr string) (int64, error) { parts := strings.Fields(sizeStr) if len(parts) != 2 { return 0, fmt.Errorf("invalid size format: %s", sizeStr) } value, err := strconv.ParseFloat(parts[0], 64) if err != nil { return 0, fmt.Errorf("invalid size value: %w", err) } unit := strings.ToUpper(parts[1]) switch unit { case "B": return int64(value), nil case "KB", "KIB": return int64(value * 1024), nil case "MB", "MIB": return int64(value * 1024 * 1024), nil case "GB", "GIB": return int64(value * 1024 * 1024 * 1024), nil case "TB", "TIB": return int64(value * 1024 * 1024 * 1024 * 1024), nil default: return 0, fmt.Errorf("unknown size unit: %s", unit) } } // RestoreBackup restores a backup with the given ID func (s *KopiaStrategy) RestoreBackup(ctx context.Context, backupID string, serviceName string) error { // Ensure repository is connected err := s.EnsureRepositoryConnected(ctx, serviceName) if err != nil { return fmt.Errorf("failed to connect to repository: %w", err) } // Parse service group and index from service name groupName, _, err := parseServiceName(serviceName) if err != nil { return fmt.Errorf("invalid service name format: %w", err) } // Get service directory from config factory, err := NewBackupFactory("config.yml") if err != nil { return fmt.Errorf("failed to create factory: %w", err) } // Find service group serviceGroup, exists := factory.Config.Services[groupName] if !exists { return fmt.Errorf("service group not found: %s", groupName) } // Create a temporary directory for restore restoreDir := filepath.Join(os.TempDir(), fmt.Sprintf("backea-restore-%s-%d", serviceName, time.Now().Unix())) if err := os.MkdirAll(restoreDir, 0755); err != nil { return fmt.Errorf("failed to create restore directory: %w", err) } log.Printf("Restoring backup %s to temporary directory %s", backupID, restoreDir) // Run kopia restore command to restore the snapshot to the temporary directory restoreCmd := exec.CommandContext( ctx, "kopia", "--config-file", s.ConfigPath, "snapshot", "restore", backupID, restoreDir, ) restoreOutput, err := restoreCmd.CombinedOutput() if err != nil { return fmt.Errorf("failed to restore snapshot: %w\nOutput: %s", err, restoreOutput) } // Now we need to sync the restored data to the original directory targetDir := serviceGroup.Source.Path log.Printf("Syncing restored data from %s to %s", restoreDir, targetDir) // Use rsync for the final transfer to avoid permissions issues syncCmd := exec.CommandContext( ctx, "rsync", "-av", "--delete", // Delete extraneous files from target restoreDir+"/", // Source directory with trailing slash to copy contents targetDir, // Target directory ) syncOutput, err := syncCmd.CombinedOutput() if err != nil { return fmt.Errorf("failed to sync restored data: %w\nOutput: %s", err, syncOutput) } // Clean up the temporary directory go func() { time.Sleep(5 * time.Minute) // Wait 5 minutes before cleaning up log.Printf("Cleaning up temporary restore directory %s", restoreDir) os.RemoveAll(restoreDir) }() log.Printf("Successfully restored backup %s to %s", backupID, targetDir) return nil } // DownloadBackup provides a reader with backup summary information in text format func (s *KopiaStrategy) DownloadBackup(ctx context.Context, backupID string, serviceName string) (io.ReadCloser, error) { // Ensure repository is connected err := s.EnsureRepositoryConnected(ctx, serviceName) if err != nil { return nil, fmt.Errorf("failed to connect to repository: %w", err) } // Create temporary directories for restore and ZIP creation tempDir, err := os.MkdirTemp("", "backea-download-*") if err != nil { return nil, fmt.Errorf("failed to create temporary directory: %w", err) } restoreDir := filepath.Join(tempDir, "restore") zipFile := filepath.Join(tempDir, "backup.zip") if err := os.MkdirAll(restoreDir, 0755); err != nil { os.RemoveAll(tempDir) return nil, fmt.Errorf("failed to create restore directory: %w", err) } // Restore the snapshot to the temporary directory using the proper snapshot ID log.Printf("Restoring snapshot %s to temporary directory %s", backupID, restoreDir) restoreCmd := exec.CommandContext( ctx, "kopia", "--config-file", s.ConfigPath, "snapshot", "restore", backupID, restoreDir, ) restoreOutput, err := restoreCmd.CombinedOutput() if err != nil { os.RemoveAll(tempDir) return nil, fmt.Errorf("failed to restore snapshot: %w\nOutput: %s", err, restoreOutput) } // Create ZIP archive of the restored files log.Printf("Creating ZIP archive at %s", zipFile) // Use Go's zip package instead of command line tools zipWriter, err := os.Create(zipFile) if err != nil { os.RemoveAll(tempDir) return nil, fmt.Errorf("failed to create zip file: %w", err) } defer zipWriter.Close() // Create ZIP writer archive := zip.NewWriter(zipWriter) defer archive.Close() // Walk the restore directory and add files to ZIP err = filepath.Walk(restoreDir, func(path string, info os.FileInfo, err error) error { if err != nil { return err } // Skip directories, we only want to add files if info.IsDir() { return nil } // Create ZIP header header, err := zip.FileInfoHeader(info) if err != nil { return err } // Make the path relative to the restore directory relPath, err := filepath.Rel(restoreDir, path) if err != nil { return err } // Set the name in the archive to the relative path header.Name = relPath // Create the file in the ZIP writer, err := archive.CreateHeader(header) if err != nil { return err } // Open the file file, err := os.Open(path) if err != nil { return err } defer file.Close() // Copy the file content to the ZIP _, err = io.Copy(writer, file) return err }) // Close the ZIP writer archive.Close() zipWriter.Close() if err != nil { os.RemoveAll(tempDir) return nil, fmt.Errorf("failed to create zip archive: %w", err) } // Open the ZIP file for reading zipReader, err := os.Open(zipFile) if err != nil { os.RemoveAll(tempDir) return nil, fmt.Errorf("failed to open zip archive: %w", err) } // Return a reader that will clean up when closed return &cleanupReadCloser{ ReadCloser: zipReader, cleanup: func() { zipReader.Close() os.RemoveAll(tempDir) }, }, nil } // GetBackupInfo returns detailed information about a specific backup func (s *KopiaStrategy) GetBackupInfo(ctx context.Context, backupID string, serviceName string) (*BackupInfo, error) { err := s.EnsureRepositoryConnected(ctx, serviceName) if err != nil { return nil, fmt.Errorf("failed to connect to repository: %w", err) } // Run kopia snapshot describe command with JSON output cmd := exec.CommandContext( ctx, "kopia", "--config-file", s.ConfigPath, "snapshot", "describe", "--json", backupID, ) output, err := cmd.Output() if err != nil { return nil, fmt.Errorf("failed to describe snapshot: %w", err) } // Parse the JSON output var snap map[string]interface{} if err := json.Unmarshal(output, &snap); err != nil { return nil, fmt.Errorf("failed to parse snapshot info: %w", err) } // Extract the relevant information id, _ := snap["id"].(string) endTime, _ := snap["endTime"].(string) // Parse time var creationTime time.Time if endTime != "" { t, err := time.Parse(time.RFC3339, endTime) if err == nil { creationTime = t } } // Get size information from stats var size int64 if stats, ok := snap["stats"].(map[string]interface{}); ok { if totalSize, ok := stats["totalSize"].(float64); ok { size = int64(totalSize) } } // Get source path sourcePath := "" if source, ok := snap["source"].(map[string]interface{}); ok { if path, ok := source["path"].(string); ok { sourcePath = path } } // Determine retention tag retentionTag := "none" if reasons, ok := snap["retentionReason"].([]interface{}); ok && len(reasons) > 0 { // Get the first reason which indicates the highest priority retention if reason, ok := reasons[0].(string); ok { parts := strings.SplitN(reason, "-", 2) if len(parts) > 0 { retentionTag = parts[0] } } } return &BackupInfo{ ID: id, CreationTime: creationTime, Size: size, Source: sourcePath, Type: "kopia", RetentionTag: retentionTag, }, nil } // cleanupReadCloser is a wrapper around io.ReadCloser that performs cleanup when closed type cleanupReadCloser struct { io.ReadCloser cleanup func() } // Close closes the underlying ReadCloser and performs cleanup func (c *cleanupReadCloser) Close() error { err := c.ReadCloser.Close() c.cleanup() return err } // extractServiceNameFromBackupID extracts the service name from a backup ID // This is an approximation as the exact format depends on your ID structure func extractServiceNameFromBackupID(backupID string) string { // Kopia snapshot IDs don't directly include the service name // You may need to adjust this based on your actual ID format // Try to extract the pattern from your backups list // If you're using ListBackups to find all backups, you might have // a mapping of IDs to service names that you can use // For this example, let's assume we're using environment variables to track this envVar := fmt.Sprintf("BACKEA_SNAPSHOT_%s", backupID) if serviceName := os.Getenv(envVar); serviceName != "" { return serviceName } // Fallback: For testing, we'll return a default or parse from the first part // This should be replaced with your actual logic parts := strings.Split(backupID, "-") if len(parts) > 0 { return parts[0] } return "unknown" }