Harden settings tunnel supervision

Author: brainplusplus

internal/tunnel/manager.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"fmt"
 	"log"
+	"runtime"
 	"sort"
 	"strconv"
 	"strings"
@@ -17,6 +18,13 @@ const (
 	StatusStopped  = "stopped"
 )
 
+var settingsTunnelRetryBackoff = []time.Duration{
+	2 * time.Second,
+	5 * time.Second,
+	10 * time.Second,
+	30 * time.Second,
+}
+
 type ConfigRecord struct {
 	ID        string `json:"id"`
 	Name      string `json:"name"`
@@ -63,6 +71,7 @@ type Manager struct {
 	appTunnelEnabled bool
 	defaultEngine    string
 	tunnels          map[string]*managedTunnel
+	retryBackoff     []time.Duration
 }
 
 func NewManager(store Store, appPort string, appTunnelEnabled bool, defaultEngine string) *Manager {
@@ -83,6 +92,7 @@ func NewManagerWithStarter(store Store, appPort string, appTunnelEnabled bool, d
 		appTunnelEnabled: appTunnelEnabled,
 		defaultEngine:    engine,
 		tunnels:          make(map[string]*managedTunnel),
+		retryBackoff:     settingsTunnelRetryBackoff,
 	}
 }
 
@@ -393,8 +403,16 @@ func (m *Manager) startAsync(id string) {
 		_ = runtimeToStop.Stop()
 	}
 
-	go func() {
+	go m.startLoop(id, seq, cfg)
+}
+
+func (m *Manager) startLoop(id string, seq uint64, cfg ConfigRecord) {
+	attempt := 0
+	for {
 		runtime, err := m.startTunnel(cfg.Engine, cfg.LocalPort)
+		if err == nil && runtime == nil {
+			err = errors.New("tunnel starter returned nil runtime")
+		}
 
 		m.mu.Lock()
 		current, ok := m.tunnels[id]
@@ -405,19 +423,59 @@ func (m *Manager) startAsync(id string) {
 			}
 			return
 		}
-		if err != nil {
-			current.status = StatusStopped
-			current.lastError = err.Error()
+		if err == nil {
+			current.runtime = runtime
+			current.status = StatusStarted
+			current.lastError = ""
 			m.mu.Unlock()
-			log.Printf("tunnel: failed to start %s on port %s: %v", current.config.Name, current.config.LocalPort, err)
 			return
 		}
 
-		current.runtime = runtime
-		current.status = StatusStarted
-		current.lastError = ""
+		current.runtime = nil
+		current.status = StatusStarting
+		current.lastError = err.Error()
+		name := current.config.Name
+		port := current.config.LocalPort
+		delay := retryDelay(m.retryBackoff, attempt)
+		attempt++
 		m.mu.Unlock()
-	}()
+
+		if runtime != nil {
+			_ = runtime.Stop()
+		}
+		log.Printf("tunnel: failed to start %s on port %s: %v; retrying in %s", name, port, err, delay)
+		if !m.waitForRetry(id, seq, delay) {
+			return
+		}
+	}
+}
+
+func (m *Manager) waitForRetry(id string, seq uint64, delay time.Duration) bool {
+	if delay <= 0 {
+		runtime.Gosched()
+		return m.shouldContinueStart(id, seq)
+	}
+	timer := time.NewTimer(delay)
+	defer timer.Stop()
+	<-timer.C
+	return m.shouldContinueStart(id, seq)
+}
+
+func (m *Manager) shouldContinueStart(id string, seq uint64) bool {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	current, ok := m.tunnels[id]
+	return ok && current.launchSeq == seq && current.config.Enabled
+}
+
+func retryDelay(backoff []time.Duration, attempt int) time.Duration {
+	if len(backoff) == 0 {
+		return time.Second
+	}
+	if attempt < len(backoff) {
+		return backoff[attempt]
+	}
+	return backoff[len(backoff)-1]
 }
 
 func (m *Manager) snapshotPortsLocked(excludeID string) map[string]string {

internal/tunnel/manager_test.go

@@ -143,3 +143,86 @@ func TestManagerLoadsEnabledTunnels(t *testing.T) {
 
 	t.Fatal("expected enabled tunnel to auto-start")
 }
+
+func TestManagerRetriesEnabledTunnelUntilStartSucceeds(t *testing.T) {
+	store := &fakeStore{}
+	startCalls := 0
+	manager := NewManagerWithStarter(store, "8080", false, "cloudflare", func(engine, port string) (RuntimeTunnel, error) {
+		startCalls++
+		if startCalls < 3 {
+			return nil, errors.New("temporary tunnel failure")
+		}
+		return &fakeRuntime{url: "https://retry.trycloudflare.com"}, nil
+	})
+	manager.retryBackoff = []time.Duration{5 * time.Millisecond}
+
+	record, err := manager.Save(ConfigRecord{
+		Name:      "Retrying Preview",
+		LocalPort: "5173",
+	})
+	if err != nil {
+		t.Fatalf("Save returned error: %v", err)
+	}
+	if _, err := manager.Start(record.ID); err != nil {
+		t.Fatalf("Start returned error: %v", err)
+	}
+
+	deadline := time.Now().Add(time.Second)
+	for time.Now().Before(deadline) {
+		current, ok := manager.Get(record.ID)
+		if ok && current.Status == StatusStarted && current.URL == "https://retry.trycloudflare.com" {
+			if startCalls != 3 {
+				t.Fatalf("expected 3 start attempts, got %d", startCalls)
+			}
+			return
+		}
+		time.Sleep(5 * time.Millisecond)
+	}
+
+	t.Fatalf("expected tunnel to retry until started; attempts=%d", startCalls)
+}
+
+func TestManagerStopCancelsPendingRetry(t *testing.T) {
+	store := &fakeStore{}
+	started := make(chan struct{}, 1)
+	startCalls := 0
+	manager := NewManagerWithStarter(store, "8080", false, "cloudflare", func(engine, port string) (RuntimeTunnel, error) {
+		startCalls++
+		started <- struct{}{}
+		return nil, errors.New("temporary tunnel failure")
+	})
+	manager.retryBackoff = []time.Duration{25 * time.Millisecond}
+
+	record, err := manager.Save(ConfigRecord{
+		Name:      "Stop Retry",
+		LocalPort: "3333",
+	})
+	if err != nil {
+		t.Fatalf("Save returned error: %v", err)
+	}
+	if _, err := manager.Start(record.ID); err != nil {
+		t.Fatalf("Start returned error: %v", err)
+	}
+
+	select {
+	case <-started:
+	case <-time.After(time.Second):
+		t.Fatal("expected first start attempt")
+	}
+
+	if _, err := manager.Stop(record.ID); err != nil {
+		t.Fatalf("Stop returned error: %v", err)
+	}
+	time.Sleep(75 * time.Millisecond)
+
+	if startCalls != 1 {
+		t.Fatalf("expected stop to cancel retry loop, got %d start attempts", startCalls)
+	}
+	current, ok := manager.Get(record.ID)
+	if !ok {
+		t.Fatal("expected tunnel record")
+	}
+	if current.Status != StatusStopped {
+		t.Fatalf("expected stopped status, got %q", current.Status)
+	}
+}