From feeeba5cee6e52aeb85edbfb63d2407f8917f50b Mon Sep 17 00:00:00 2001 From: Milo Schwartz Date: Tue, 4 Feb 2025 22:46:41 -0500 Subject: [PATCH 01/26] fix path in cicd --- .github/workflows/cicd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 65b01b26..be0fc303 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -36,7 +36,7 @@ jobs: run: | TAG=${{ env.TAG }} sed -i "s/export const APP_VERSION = \".*\";/export const APP_VERSION = \"$TAG\";/" server/lib/consts.ts - cat server/lib/ + cat server/lib/consts.ts - name: Pull latest Gerbil version id: get-gerbil-tag From d22c7826fe63ab58e44988597f3df260192507c8 Mon Sep 17 00:00:00 2001 From: Owen Date: Sun, 9 Feb 2025 16:14:29 -0500 Subject: [PATCH 02/26] Add config files --- install/captcha.html | 338 ++++++++++++++++ install/crowdsec/install.go | 376 ++++++++++++++++++ install/fs/crowdsec/acquis.yaml | 18 + install/fs/crowdsec/config.yaml | 12 + .../fs/crowdsec/local_api_credentials.yaml | 2 + install/fs/crowdsec/profiles.yaml | 25 ++ 6 files changed, 771 insertions(+) create mode 100644 install/captcha.html create mode 100644 install/crowdsec/install.go create mode 100644 install/fs/crowdsec/acquis.yaml create mode 100644 install/fs/crowdsec/config.yaml create mode 100644 install/fs/crowdsec/local_api_credentials.yaml create mode 100644 install/fs/crowdsec/profiles.yaml diff --git a/install/captcha.html b/install/captcha.html new file mode 100644 index 00000000..a40d37a3 --- /dev/null +++ b/install/captcha.html @@ -0,0 +1,338 @@ + + + + + CrowdSec Captcha + + + + + + + +
+
+
+ +

CrowdSec Captcha

+
+
+
+
+
+
+

This security check has been powered by

+ + + + + + + + + + + + + + + + + + + + + CrowdSec + +
+
+
+ + + \ No newline at end of file diff --git a/install/crowdsec/install.go b/install/crowdsec/install.go new file mode 100644 index 00000000..84f3089b --- /dev/null +++ b/install/crowdsec/install.go @@ -0,0 +1,376 @@ +package crowdsec + +import ( + "bytes" + "embed" + "encoding/json" + "fmt" + "io" + "os" + "os/exec" + "strings" + "time" +) + +//go:embed fs/* +var configFiles embed.FS + +// Config holds all configuration values +type Config struct { + DomainName string + EnrollmentKey string + TurnstileSiteKey string + TurnstileSecretKey string + GID string + CrowdsecIP string + TraefikBouncerKey string + PangolinIP string +} + +// DockerContainer represents a Docker container +type DockerContainer struct { + NetworkSettings struct { + Networks map[string]struct { + IPAddress string `json:"IPAddress"` + } `json:"Networks"` + } `json:"NetworkSettings"` +} + +func main() { + if err := run(); err != nil { + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + os.Exit(1) + } +} + +func run() error { + // Create configuration + config := &Config{} + + // Run installation steps + if err := backupConfig(); err != nil { + return fmt.Errorf("backup failed: %v", err) + } + + if err := createPangolinNetwork(); err != nil { + return fmt.Errorf("network creation failed: %v", err) + } + + if err := modifyDockerCompose(); err != nil { + return fmt.Errorf("docker-compose modification failed: %v", err) + } + + if err := createConfigFiles(*config); err != nil { + return fmt.Errorf("config file creation failed: %v", err) + } + + if err := retrieveIPs(config); err != nil { + return fmt.Errorf("IP retrieval failed: %v", err) + } + + if err := retrieveBouncerKey(config); err != nil { + return fmt.Errorf("bouncer key retrieval failed: %v", err) + } + + if err := replacePlaceholders(config); err != nil { + return fmt.Errorf("placeholder replacement failed: %v", err) + } + + if err := deployStack(); err != nil { + return fmt.Errorf("deployment failed: %v", err) + } + + if err := verifyDeployment(); err != nil { + return fmt.Errorf("verification failed: %v", err) + } + + printInstructions() + return nil +} + +func backupConfig() error { + // Backup docker-compose.yml + if _, err := os.Stat("docker-compose.yml"); err == nil { + if err := copyFile("docker-compose.yml", "docker-compose.yml.backup"); err != nil { + return fmt.Errorf("failed to backup docker-compose.yml: %v", err) + } + } + + // Backup config directory + if _, err := os.Stat("config"); err == nil { + cmd := exec.Command("tar", "-czvf", "config.tar.gz", "config") + if err := cmd.Run(); err != nil { + return fmt.Errorf("failed to backup config directory: %v", err) + } + } + + return nil +} + +func createPangolinNetwork() error { + // Check if network exists + cmd := exec.Command("docker", "network", "inspect", "pangolin") + if err := cmd.Run(); err == nil { + fmt.Println("pangolin network already exists") + return nil + } + + // Create network + cmd = exec.Command("docker", "network", "create", "pangolin") + if err := cmd.Run(); err != nil { + return fmt.Errorf("failed to create pangolin network: %v", err) + } + + return nil +} + +func modifyDockerCompose() error { + // Read existing docker-compose.yml + content, err := os.ReadFile("docker-compose.yml") + if err != nil { + return fmt.Errorf("failed to read docker-compose.yml: %v", err) + } + + // Verify required services exist + requiredServices := []string{"services:", "pangolin:", "gerbil:", "traefik:"} + for _, service := range requiredServices { + if !bytes.Contains(content, []byte(service)) { + return fmt.Errorf("required service %s not found in docker-compose.yml", service) + } + } + + // Add crowdsec service + modified := addCrowdsecService(string(content)) + + // Write modified content + if err := os.WriteFile("docker-compose.yml", []byte(modified), 0644); err != nil { + return fmt.Errorf("failed to write modified docker-compose.yml: %v", err) + } + + return nil +} + +func retrieveIPs(config *Config) error { + // Start required containers + cmd := exec.Command("docker", "compose", "up", "-d", "pangolin", "crowdsec") + if err := cmd.Run(); err != nil { + return fmt.Errorf("failed to start containers: %v", err) + } + defer exec.Command("docker", "compose", "down").Run() + + // Wait for containers to start + time.Sleep(10 * time.Second) + + // Get Pangolin IP + pangolinIP, err := getContainerIP("pangolin") + if err != nil { + return fmt.Errorf("failed to get pangolin IP: %v", err) + } + config.PangolinIP = pangolinIP + + // Get CrowdSec IP + crowdsecIP, err := getContainerIP("crowdsec") + if err != nil { + return fmt.Errorf("failed to get crowdsec IP: %v", err) + } + config.CrowdsecIP = crowdsecIP + + return nil +} + +func retrieveBouncerKey(config *Config) error { + // Start crowdsec container + cmd := exec.Command("docker", "compose", "up", "-d", "crowdsec") + if err := cmd.Run(); err != nil { + return fmt.Errorf("failed to start crowdsec: %v", err) + } + defer exec.Command("docker", "compose", "down").Run() + + // Wait for container to start + time.Sleep(10 * time.Second) + + // Get bouncer key + output, err := exec.Command("docker", "exec", "crowdsec", "cscli", "bouncers", "add", "traefik-bouncer").Output() + if err != nil { + return fmt.Errorf("failed to get bouncer key: %v", err) + } + + // Parse key from output + lines := strings.Split(string(output), "\n") + for _, line := range lines { + if strings.Contains(line, "key:") { + config.TraefikBouncerKey = strings.TrimSpace(strings.Split(line, ":")[1]) + break + } + } + + return nil +} + +func replacePlaceholders(config *Config) error { + // Get user input + fmt.Print("Enter your Domain Name (e.g., pangolin.example.com): ") + fmt.Scanln(&config.DomainName) + + fmt.Print("Enter your CrowdSec Enrollment Key: ") + fmt.Scanln(&config.EnrollmentKey) + + fmt.Print("Enter your Cloudflare Turnstile Site Key: ") + fmt.Scanln(&config.TurnstileSiteKey) + + fmt.Print("Enter your Cloudflare Turnstile Secret Key: ") + fmt.Scanln(&config.TurnstileSecretKey) + + fmt.Print("Enter your GID (or leave empty for default 1000): ") + gid := "" + fmt.Scanln(&gid) + if gid == "" { + config.GID = "1000" + } else { + config.GID = gid + } + + return nil +} + +func deployStack() error { + cmd := exec.Command("docker", "compose", "up", "-d") + if err := cmd.Run(); err != nil { + return fmt.Errorf("failed to deploy stack: %v", err) + } + + fmt.Println("Stack deployed. Waiting 2 minutes for services to initialize...") + time.Sleep(2 * time.Minute) + return nil +} + +func verifyDeployment() error { + resp, err := exec.Command("curl", "-s", "http://localhost:6060/metrics").Output() + if err != nil { + return fmt.Errorf("failed to get metrics: %v", err) + } + + if !bytes.Contains(resp, []byte("appsec")) { + return fmt.Errorf("appsec metrics not found in response") + } + + return nil +} + +func printInstructions() { + fmt.Println(` +--- Testing Instructions --- +1. Test Captcha Implementation: + docker exec crowdsec cscli decisions add --ip YOUR_IP --type captcha -d 1h + (Replace YOUR_IP with your actual IP address) + +2. Verify decisions: + docker exec -it crowdsec cscli decisions list + +3. Test security by accessing DOMAIN_NAME/.env (should return 403) + (Replace DOMAIN_NAME with the domain you entered) + +--- Troubleshooting --- +1. If encountering 403 errors: + - Check Traefik logs: docker compose logs traefik -f + - Verify CrowdSec logs: docker compose logs crowdsec + +2. For plugin errors: + - Verify http notifications are commented out in profiles.yaml + - Restart services: docker compose restart traefik crowdsec + +3. For Captcha issues: + - Ensure Turnstile is configured in non-interactive mode + - Verify captcha.html configuration + - Check container network connectivity + +Useful Commands: +- View Traefik logs: docker compose logs traefik -f +- View CrowdSec logs: docker compose logs crowdsec +- List decisions: docker exec -it crowdsec cscli decisions list +- Check metrics: curl http://localhost:6060/metrics | grep appsec +`) +} + +// Helper functions + +func copyFile(src, dst string) error { + source, err := os.Open(src) + if err != nil { + return err + } + defer source.Close() + + destination, err := os.Create(dst) + if err != nil { + return err + } + defer destination.Close() + + _, err = io.Copy(destination, source) + return err +} + +func getContainerIP(containerName string) (string, error) { + output, err := exec.Command("docker", "inspect", containerName).Output() + if err != nil { + return "", err + } + + var containers []DockerContainer + if err := json.Unmarshal(output, &containers); err != nil { + return "", err + } + + if len(containers) == 0 { + return "", fmt.Errorf("no container found") + } + + for _, network := range containers[0].NetworkSettings.Networks { + return network.IPAddress, nil + } + + return "", fmt.Errorf("no IP address found") +} + +func addCrowdsecService(content string) string { + // Implementation of adding crowdsec service to docker-compose.yml + // This would involve string manipulation or template rendering + // The actual implementation would depend on how you want to structure the docker-compose modifications + return content + ` + crowdsec: + image: crowdsecurity/crowdsec:latest + container_name: crowdsec + environment: + GID: "${GID-1000}" + COLLECTIONS: crowdsecurity/traefik crowdsecurity/appsec-virtual-patching crowdsecurity/appsec-generic-rules + ENROLL_INSTANCE_NAME: "pangolin-crowdsec" + PARSERS: crowdsecurity/whitelists + ENROLL_KEY: ${ENROLLMENT_KEY} + ACQUIRE_FILES: "/var/log/traefik/*.log" + ENROLL_TAGS: docker + networks: + - pangolin + healthcheck: + test: ["CMD", "cscli", "capi", "status"] + depends_on: + - gerbil + labels: + - "traefik.enable=false" + volumes: + - ./config/crowdsec:/etc/crowdsec + - ./config/crowdsec/db:/var/lib/crowdsec/data + - ./config/crowdsec_logs/auth.log:/var/log/auth.log:ro + - ./config/crowdsec_logs/syslog:/var/log/syslog:ro + - ./config/crowdsec_logs:/var/log + - ./config/traefik/logs:/var/log/traefik + ports: + - 9090:9090 + - 6060:6060 + expose: + - 9090 + - 6060 + - 7422 + restart: unless-stopped + command: -t` +} diff --git a/install/fs/crowdsec/acquis.yaml b/install/fs/crowdsec/acquis.yaml new file mode 100644 index 00000000..74d8fd1c --- /dev/null +++ b/install/fs/crowdsec/acquis.yaml @@ -0,0 +1,18 @@ +filenames: + - /var/log/auth.log + - /var/log/syslog +labels: + type: syslog +--- +poll_without_inotify: false +filenames: + - /var/log/traefik/*.log +labels: + type: traefik +--- +listen_addr: 0.0.0.0:7422 +appsec_config: crowdsecurity/appsec-default +name: myAppSecComponent +source: appsec +labels: + type: appsec \ No newline at end of file diff --git a/install/fs/crowdsec/config.yaml b/install/fs/crowdsec/config.yaml new file mode 100644 index 00000000..0acf4635 --- /dev/null +++ b/install/fs/crowdsec/config.yaml @@ -0,0 +1,12 @@ +api: + client: + insecure_skip_verify: false + credentials_path: /etc/crowdsec/local_api_credentials.yaml + server: + log_level: info + listen_uri: 0.0.0.0:9090 + profiles_path: /etc/crowdsec/profiles.yaml + trusted_ips: + - 0.0.0.0/0 + - 127.0.0.1 + - ::1 \ No newline at end of file diff --git a/install/fs/crowdsec/local_api_credentials.yaml b/install/fs/crowdsec/local_api_credentials.yaml new file mode 100644 index 00000000..8776e4fd --- /dev/null +++ b/install/fs/crowdsec/local_api_credentials.yaml @@ -0,0 +1,2 @@ +url: http://0.0.0.0:9090 +login: localhost \ No newline at end of file diff --git a/install/fs/crowdsec/profiles.yaml b/install/fs/crowdsec/profiles.yaml new file mode 100644 index 00000000..3796b47f --- /dev/null +++ b/install/fs/crowdsec/profiles.yaml @@ -0,0 +1,25 @@ +name: captcha_remediation +filters: + - Alert.Remediation == true && Alert.GetScope() == "Ip" && Alert.GetScenario() contains "http" +decisions: + - type: captcha + duration: 4h +on_success: break + +--- +name: default_ip_remediation +filters: + - Alert.Remediation == true && Alert.GetScope() == "Ip" +decisions: + - type: ban + duration: 4h +on_success: break + +--- +name: default_range_remediation +filters: + - Alert.Remediation == true && Alert.GetScope() == "Range" +decisions: + - type: ban + duration: 4h +on_success: break \ No newline at end of file From 81c4199e87fdd066495101f85e47bcb6f49a1058 Mon Sep 17 00:00:00 2001 From: Owen Date: Wed, 12 Feb 2025 21:56:13 -0500 Subject: [PATCH 03/26] Complete bash migration --- install/captcha.html | 338 ------------------ install/{crowdsec/install.go => crowdsec.go} | 271 ++++++-------- install/{fs => }/crowdsec/acquis.yaml | 0 install/{fs => }/crowdsec/config.yaml | 0 install/crowdsec/dynamic_config.yml | 108 ++++++ .../crowdsec/local_api_credentials.yaml | 0 install/{fs => }/crowdsec/profiles.yaml | 0 install/crowdsec/traefik_config.yml | 87 +++++ install/main.go | 1 + 9 files changed, 301 insertions(+), 504 deletions(-) delete mode 100644 install/captcha.html rename install/{crowdsec/install.go => crowdsec.go} (55%) rename install/{fs => }/crowdsec/acquis.yaml (100%) rename install/{fs => }/crowdsec/config.yaml (100%) create mode 100644 install/crowdsec/dynamic_config.yml rename install/{fs => }/crowdsec/local_api_credentials.yaml (100%) rename install/{fs => }/crowdsec/profiles.yaml (100%) create mode 100644 install/crowdsec/traefik_config.yml diff --git a/install/captcha.html b/install/captcha.html deleted file mode 100644 index a40d37a3..00000000 --- a/install/captcha.html +++ /dev/null @@ -1,338 +0,0 @@ - - - - - CrowdSec Captcha - - - - - - - -
-
-
- -

CrowdSec Captcha

-
-
-
-
-
-
-

This security check has been powered by

- - - - - - - - - - - - - - - - - - - - - CrowdSec - -
-
-
- - - \ No newline at end of file diff --git a/install/crowdsec/install.go b/install/crowdsec.go similarity index 55% rename from install/crowdsec/install.go rename to install/crowdsec.go index 84f3089b..187fea88 100644 --- a/install/crowdsec/install.go +++ b/install/crowdsec.go @@ -1,31 +1,21 @@ -package crowdsec +package main import ( "bytes" "embed" - "encoding/json" "fmt" + "html/template" "io" + "io/fs" "os" "os/exec" + "path/filepath" "strings" "time" ) -//go:embed fs/* -var configFiles embed.FS - -// Config holds all configuration values -type Config struct { - DomainName string - EnrollmentKey string - TurnstileSiteKey string - TurnstileSecretKey string - GID string - CrowdsecIP string - TraefikBouncerKey string - PangolinIP string -} +//go:embed crowdsec/* +var configCrowdsecFiles embed.FS // DockerContainer represents a Docker container type DockerContainer struct { @@ -36,14 +26,7 @@ type DockerContainer struct { } `json:"NetworkSettings"` } -func main() { - if err := run(); err != nil { - fmt.Fprintf(os.Stderr, "Error: %v\n", err) - os.Exit(1) - } -} - -func run() error { +func installCrowdsec() error { // Create configuration config := &Config{} @@ -52,30 +35,21 @@ func run() error { return fmt.Errorf("backup failed: %v", err) } - if err := createPangolinNetwork(); err != nil { - return fmt.Errorf("network creation failed: %v", err) - } - if err := modifyDockerCompose(); err != nil { return fmt.Errorf("docker-compose modification failed: %v", err) } - if err := createConfigFiles(*config); err != nil { + if err := createCrowdsecFiles(*config); err != nil { return fmt.Errorf("config file creation failed: %v", err) } - if err := retrieveIPs(config); err != nil { - return fmt.Errorf("IP retrieval failed: %v", err) - } + moveFile("config/crowdsec/traefik_config.yaml", "config/traefik/traefik_config.yaml") + moveFile("config/crowdsec/dynamic.yaml", "config/traefik/dynamic.yaml") if err := retrieveBouncerKey(config); err != nil { return fmt.Errorf("bouncer key retrieval failed: %v", err) } - if err := replacePlaceholders(config); err != nil { - return fmt.Errorf("placeholder replacement failed: %v", err) - } - if err := deployStack(); err != nil { return fmt.Errorf("deployment failed: %v", err) } @@ -84,7 +58,6 @@ func run() error { return fmt.Errorf("verification failed: %v", err) } - printInstructions() return nil } @@ -107,23 +80,6 @@ func backupConfig() error { return nil } -func createPangolinNetwork() error { - // Check if network exists - cmd := exec.Command("docker", "network", "inspect", "pangolin") - if err := cmd.Run(); err == nil { - fmt.Println("pangolin network already exists") - return nil - } - - // Create network - cmd = exec.Command("docker", "network", "create", "pangolin") - if err := cmd.Run(); err != nil { - return fmt.Errorf("failed to create pangolin network: %v", err) - } - - return nil -} - func modifyDockerCompose() error { // Read existing docker-compose.yml content, err := os.ReadFile("docker-compose.yml") @@ -150,34 +106,6 @@ func modifyDockerCompose() error { return nil } -func retrieveIPs(config *Config) error { - // Start required containers - cmd := exec.Command("docker", "compose", "up", "-d", "pangolin", "crowdsec") - if err := cmd.Run(); err != nil { - return fmt.Errorf("failed to start containers: %v", err) - } - defer exec.Command("docker", "compose", "down").Run() - - // Wait for containers to start - time.Sleep(10 * time.Second) - - // Get Pangolin IP - pangolinIP, err := getContainerIP("pangolin") - if err != nil { - return fmt.Errorf("failed to get pangolin IP: %v", err) - } - config.PangolinIP = pangolinIP - - // Get CrowdSec IP - crowdsecIP, err := getContainerIP("crowdsec") - if err != nil { - return fmt.Errorf("failed to get crowdsec IP: %v", err) - } - config.CrowdsecIP = crowdsecIP - - return nil -} - func retrieveBouncerKey(config *Config) error { // Start crowdsec container cmd := exec.Command("docker", "compose", "up", "-d", "crowdsec") @@ -207,32 +135,6 @@ func retrieveBouncerKey(config *Config) error { return nil } -func replacePlaceholders(config *Config) error { - // Get user input - fmt.Print("Enter your Domain Name (e.g., pangolin.example.com): ") - fmt.Scanln(&config.DomainName) - - fmt.Print("Enter your CrowdSec Enrollment Key: ") - fmt.Scanln(&config.EnrollmentKey) - - fmt.Print("Enter your Cloudflare Turnstile Site Key: ") - fmt.Scanln(&config.TurnstileSiteKey) - - fmt.Print("Enter your Cloudflare Turnstile Secret Key: ") - fmt.Scanln(&config.TurnstileSecretKey) - - fmt.Print("Enter your GID (or leave empty for default 1000): ") - gid := "" - fmt.Scanln(&gid) - if gid == "" { - config.GID = "1000" - } else { - config.GID = gid - } - - return nil -} - func deployStack() error { cmd := exec.Command("docker", "compose", "up", "-d") if err := cmd.Run(); err != nil { @@ -257,43 +159,6 @@ func verifyDeployment() error { return nil } -func printInstructions() { - fmt.Println(` ---- Testing Instructions --- -1. Test Captcha Implementation: - docker exec crowdsec cscli decisions add --ip YOUR_IP --type captcha -d 1h - (Replace YOUR_IP with your actual IP address) - -2. Verify decisions: - docker exec -it crowdsec cscli decisions list - -3. Test security by accessing DOMAIN_NAME/.env (should return 403) - (Replace DOMAIN_NAME with the domain you entered) - ---- Troubleshooting --- -1. If encountering 403 errors: - - Check Traefik logs: docker compose logs traefik -f - - Verify CrowdSec logs: docker compose logs crowdsec - -2. For plugin errors: - - Verify http notifications are commented out in profiles.yaml - - Restart services: docker compose restart traefik crowdsec - -3. For Captcha issues: - - Ensure Turnstile is configured in non-interactive mode - - Verify captcha.html configuration - - Check container network connectivity - -Useful Commands: -- View Traefik logs: docker compose logs traefik -f -- View CrowdSec logs: docker compose logs crowdsec -- List decisions: docker exec -it crowdsec cscli decisions list -- Check metrics: curl http://localhost:6060/metrics | grep appsec -`) -} - -// Helper functions - func copyFile(src, dst string) error { source, err := os.Open(src) if err != nil { @@ -311,26 +176,12 @@ func copyFile(src, dst string) error { return err } -func getContainerIP(containerName string) (string, error) { - output, err := exec.Command("docker", "inspect", containerName).Output() - if err != nil { - return "", err +func moveFile(src, dst string) error { + if err := copyFile(src, dst); err != nil { + return err } - var containers []DockerContainer - if err := json.Unmarshal(output, &containers); err != nil { - return "", err - } - - if len(containers) == 0 { - return "", fmt.Errorf("no container found") - } - - for _, network := range containers[0].NetworkSettings.Networks { - return network.IPAddress, nil - } - - return "", fmt.Errorf("no IP address found") + return os.Remove(src) } func addCrowdsecService(content string) string { @@ -346,11 +197,8 @@ func addCrowdsecService(content string) string { COLLECTIONS: crowdsecurity/traefik crowdsecurity/appsec-virtual-patching crowdsecurity/appsec-generic-rules ENROLL_INSTANCE_NAME: "pangolin-crowdsec" PARSERS: crowdsecurity/whitelists - ENROLL_KEY: ${ENROLLMENT_KEY} ACQUIRE_FILES: "/var/log/traefik/*.log" ENROLL_TAGS: docker - networks: - - pangolin healthcheck: test: ["CMD", "cscli", "capi", "status"] depends_on: @@ -374,3 +222,94 @@ func addCrowdsecService(content string) string { restart: unless-stopped command: -t` } + +func createCrowdsecFiles(config Config) error { + // Walk through all embedded files + err := fs.WalkDir(configCrowdsecFiles, "crowdsec", func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + + // Skip the root fs directory itself + if path == "fs" { + return nil + } + + // Get the relative path by removing the "fs/" prefix + relPath := strings.TrimPrefix(path, "fs/") + + // skip .DS_Store + if strings.Contains(relPath, ".DS_Store") { + return nil + } + + // Create the full output path under "config/" + outPath := filepath.Join("config", relPath) + + if d.IsDir() { + // Create directory + if err := os.MkdirAll(outPath, 0755); err != nil { + return fmt.Errorf("failed to create directory %s: %v", outPath, err) + } + return nil + } + + // Read the template file + content, err := configFiles.ReadFile(path) + if err != nil { + return fmt.Errorf("failed to read %s: %v", path, err) + } + + // Parse template + tmpl, err := template.New(d.Name()).Parse(string(content)) + if err != nil { + return fmt.Errorf("failed to parse template %s: %v", path, err) + } + + // Ensure parent directory exists + if err := os.MkdirAll(filepath.Dir(outPath), 0755); err != nil { + return fmt.Errorf("failed to create parent directory for %s: %v", outPath, err) + } + + // Create output file + outFile, err := os.Create(outPath) + if err != nil { + return fmt.Errorf("failed to create %s: %v", outPath, err) + } + defer outFile.Close() + + // Execute template + if err := tmpl.Execute(outFile, config); err != nil { + return fmt.Errorf("failed to execute template %s: %v", path, err) + } + + return nil + }) + + if err != nil { + return fmt.Errorf("error walking config files: %v", err) + } + + // get the current directory + dir, err := os.Getwd() + if err != nil { + return fmt.Errorf("failed to get current directory: %v", err) + } + + sourcePath := filepath.Join(dir, "config/docker-compose.yml") + destPath := filepath.Join(dir, "docker-compose.yml") + + // Check if source file exists + if _, err := os.Stat(sourcePath); err != nil { + return fmt.Errorf("source docker-compose.yml not found: %v", err) + } + + // Try to move the file + err = os.Rename(sourcePath, destPath) + if err != nil { + return fmt.Errorf("failed to move docker-compose.yml from %s to %s: %v", + sourcePath, destPath, err) + } + + return nil +} diff --git a/install/fs/crowdsec/acquis.yaml b/install/crowdsec/acquis.yaml similarity index 100% rename from install/fs/crowdsec/acquis.yaml rename to install/crowdsec/acquis.yaml diff --git a/install/fs/crowdsec/config.yaml b/install/crowdsec/config.yaml similarity index 100% rename from install/fs/crowdsec/config.yaml rename to install/crowdsec/config.yaml diff --git a/install/crowdsec/dynamic_config.yml b/install/crowdsec/dynamic_config.yml new file mode 100644 index 00000000..9175b143 --- /dev/null +++ b/install/crowdsec/dynamic_config.yml @@ -0,0 +1,108 @@ +http: + middlewares: + redirect-to-https: + redirectScheme: + scheme: https + default-whitelist: # Whitelist middleware for internal IPs + ipWhiteList: # Internal IP addresses + sourceRange: # Internal IP addresses + - "10.0.0.0/8" # Internal IP addresses + - "192.168.0.0/16" # Internal IP addresses + - "172.16.0.0/12" # Internal IP addresses + # Basic security headers + security-headers: + headers: + customResponseHeaders: # Custom response headers + Server: "" # Remove server header + X-Powered-By: "" # Remove powered by header + X-Forwarded-Proto: "https" # Set forwarded proto to https + sslProxyHeaders: # SSL proxy headers + X-Forwarded-Proto: "https" # Set forwarded proto to https + hostsProxyHeaders: # Hosts proxy headers + - "X-Forwarded-Host" # Set forwarded host + contentTypeNosniff: true # Prevent MIME sniffing + customFrameOptionsValue: "SAMEORIGIN" # Set frame options + referrerPolicy: "strict-origin-when-cross-origin" # Set referrer policy + forceSTSHeader: true # Force STS header + stsIncludeSubdomains: true # Include subdomains + stsSeconds: 63072000 # STS seconds + stsPreload: true # Preload STS + # CrowdSec configuration with proper IP forwarding + crowdsec: + plugin: + crowdsec: + enabled: true # Enable CrowdSec plugin + logLevel: INFO # Log level + updateIntervalSeconds: 15 # Update interval + updateMaxFailure: 0 # Update max failure + defaultDecisionSeconds: 15 # Default decision seconds + httpTimeoutSeconds: 10 # HTTP timeout + crowdsecMode: live # CrowdSec mode + crowdsecAppsecEnabled: true # Enable AppSec + crowdsecAppsecHost: crowdsec:7422 # CrowdSec IP address which you noted down later + crowdsecAppsecFailureBlock: true # Block on failure + crowdsecAppsecUnreachableBlock: true # Block on unreachable + crowdsecLapiKey: "{{.TraefikBouncerKey}}" # CrowdSec API key which you noted down later + crowdsecLapiHost: crowdsec:9090 # CrowdSec + crowdsecLapiScheme: http # CrowdSec API scheme + forwardedHeadersTrustedIPs: # Forwarded headers trusted IPs + - "0.0.0.0/0" # All IP addresses are trusted for forwarded headers (CHANGE MADE HERE) + clientTrustedIPs: # Client trusted IPs (CHANGE MADE HERE) + - "10.0.0.0/8" # Internal LAN IP addresses + - "172.16.0.0/12" # Internal LAN IP addresses + - "192.168.0.0/16" # Internal LAN IP addresses + - "100.89.137.0/20" # Internal LAN IP addresses + + routers: + # HTTP to HTTPS redirect router + main-app-router-redirect: + rule: "Host(`{{.DomainName}}`)" # Dynamic Domain Name + service: next-service + entryPoints: + - web + middlewares: + - redirect-to-https + + # Next.js router (handles everything except API and WebSocket paths) + next-router: + rule: "Host(`{{.DomainName}}`) && !PathPrefix(`/api/v1`)" # Dynamic Domain Name + service: next-service + entryPoints: + - websecure + middlewares: + - security-headers # Add security headers middleware + tls: + certResolver: letsencrypt + + # API router (handles /api/v1 paths) + api-router: + rule: "Host(`{{.DomainName}}`) && PathPrefix(`/api/v1`)" # Dynamic Domain Name + service: api-service + entryPoints: + - websecure + middlewares: + - security-headers # Add security headers middleware + tls: + certResolver: letsencrypt + + # WebSocket router + ws-router: + rule: "Host(`{{.DomainName}}`)" # Dynamic Domain Name + service: api-service + entryPoints: + - websecure + middlewares: + - security-headers # Add security headers middleware + tls: + certResolver: letsencrypt + + services: + next-service: + loadBalancer: + servers: + - url: "http://pangolin:3002" # Next.js server + + api-service: + loadBalancer: + servers: + - url: "http://pangolin:3000" # API/WebSocket server \ No newline at end of file diff --git a/install/fs/crowdsec/local_api_credentials.yaml b/install/crowdsec/local_api_credentials.yaml similarity index 100% rename from install/fs/crowdsec/local_api_credentials.yaml rename to install/crowdsec/local_api_credentials.yaml diff --git a/install/fs/crowdsec/profiles.yaml b/install/crowdsec/profiles.yaml similarity index 100% rename from install/fs/crowdsec/profiles.yaml rename to install/crowdsec/profiles.yaml diff --git a/install/crowdsec/traefik_config.yml b/install/crowdsec/traefik_config.yml new file mode 100644 index 00000000..2ac9125c --- /dev/null +++ b/install/crowdsec/traefik_config.yml @@ -0,0 +1,87 @@ +api: + insecure: true + dashboard: true + +providers: + http: + endpoint: "http://pangolin:3001/api/v1/traefik-config" + pollInterval: "5s" + file: + filename: "/etc/traefik/dynamic_config.yml" + +experimental: + plugins: + badger: + moduleName: "github.com/fosrl/badger" + version: "{{.BadgerVersion}}" + crowdsec: # CrowdSec plugin configuration added + moduleName: "github.com/maxlerebourg/crowdsec-bouncer-traefik-plugin" + version: "v1.3.5" + +log: + level: "INFO" + format: "json" # Log format changed to json for better parsing + +accessLog: # We enable access logs as json + filePath: "/var/log/traefik/access.log" + format: json + filters: + statusCodes: + - "200-299" # Success codes + - "400-499" # Client errors + - "500-599" # Server errors + retryAttempts: true + minDuration: "100ms" # Increased to focus on slower requests + bufferingSize: 100 # Add buffering for better performance + fields: + defaultMode: drop # Start with dropping all fields + names: + ClientAddr: keep # Keep client address for IP tracking + ClientHost: keep # Keep client host for IP tracking + RequestMethod: keep # Keep request method for tracking + RequestPath: keep # Keep request path for tracking + RequestProtocol: keep # Keep request protocol for tracking + DownstreamStatus: keep # Keep downstream status for tracking + DownstreamContentSize: keep # Keep downstream content size for tracking + Duration: keep # Keep request duration for tracking + ServiceName: keep # Keep service name for tracking + StartUTC: keep # Keep start time for tracking + TLSVersion: keep # Keep TLS version for tracking + TLSCipher: keep # Keep TLS cipher for tracking + RetryAttempts: keep # Keep retry attempts for tracking + headers: + defaultMode: drop # Start with dropping all headers + names: + User-Agent: keep # Keep user agent for tracking + X-Real-Ip: keep # Keep real IP for tracking + X-Forwarded-For: keep # Keep forwarded IP for tracking + X-Forwarded-Proto: keep # Keep forwarded protocol for tracking + Content-Type: keep # Keep content type for tracking + Authorization: redact # Redact sensitive information + Cookie: redact # Redact sensitive information + +certificatesResolvers: + letsencrypt: + acme: + httpChallenge: + entryPoint: web + email: "{{.LetsEncryptEmail}}" + storage: "/letsencrypt/acme.json" + caServer: "https://acme-v02.api.letsencrypt.org/directory" + +entryPoints: + web: + address: ":80" + websecure: + address: ":443" + transport: + respondingTimeouts: + readTimeout: "30m" + http: + tls: + certResolver: "letsencrypt" + middlewares: # CHANGE MADE HERE (BOUNCER ENABLED) !!! + - crowdsec@file + +serversTransport: + insecureSkipVerify: true \ No newline at end of file diff --git a/install/main.go b/install/main.go index 4f2deb3a..d5bf5e1e 100644 --- a/install/main.go +++ b/install/main.go @@ -45,6 +45,7 @@ type Config struct { EmailSMTPPass string EmailNoReply string InstallGerbil bool + TraefikBouncerKey string } func main() { From 60449afca5f3186e31c5d8210e116c9a49de884b Mon Sep 17 00:00:00 2001 From: Owen Date: Thu, 13 Feb 2025 22:16:52 -0500 Subject: [PATCH 04/26] Reorg; create crowdsec folder properly now --- install/config.go | 216 ++++++++++++++ install/{fs => config}/config.yml | 0 install/{ => config}/crowdsec/acquis.yaml | 0 install/{ => config}/crowdsec/config.yaml | 0 .../{ => config}/crowdsec/dynamic_config.yml | 8 +- .../crowdsec/local_api_credentials.yaml | 0 install/{ => config}/crowdsec/profiles.yaml | 0 .../{ => config}/crowdsec/traefik_config.yml | 0 install/{fs => config}/docker-compose.yml | 5 +- .../{fs => config}/traefik/dynamic_config.yml | 0 .../{fs => config}/traefik/traefik_config.yml | 0 install/crowdsec.go | 267 ++---------------- install/go.mod | 1 + install/go.sum | 3 + install/main.go | 134 ++++++--- 15 files changed, 349 insertions(+), 285 deletions(-) create mode 100644 install/config.go rename install/{fs => config}/config.yml (100%) rename install/{ => config}/crowdsec/acquis.yaml (100%) rename install/{ => config}/crowdsec/config.yaml (100%) rename install/{ => config}/crowdsec/dynamic_config.yml (92%) rename install/{ => config}/crowdsec/local_api_credentials.yaml (100%) rename install/{ => config}/crowdsec/profiles.yaml (100%) rename install/{ => config}/crowdsec/traefik_config.yml (100%) rename install/{fs => config}/docker-compose.yml (97%) rename install/{fs => config}/traefik/dynamic_config.yml (100%) rename install/{fs => config}/traefik/traefik_config.yml (100%) diff --git a/install/config.go b/install/config.go new file mode 100644 index 00000000..fa10fa53 --- /dev/null +++ b/install/config.go @@ -0,0 +1,216 @@ +package main + +import ( + "bytes" + "fmt" + "os" + + "gopkg.in/yaml.v3" +) + +// TraefikConfig represents the structure of the main Traefik configuration +type TraefikConfig struct { + Experimental struct { + Plugins struct { + Badger struct { + Version string `yaml:"version"` + } `yaml:"badger"` + } `yaml:"plugins"` + } `yaml:"experimental"` + CertificatesResolvers struct { + LetsEncrypt struct { + Acme struct { + Email string `yaml:"email"` + } `yaml:"acme"` + } `yaml:"letsencrypt"` + } `yaml:"certificatesResolvers"` +} + +// DynamicConfig represents the structure of the dynamic configuration +type DynamicConfig struct { + HTTP struct { + Routers map[string]struct { + Rule string `yaml:"rule"` + } `yaml:"routers"` + } `yaml:"http"` +} + +// ConfigValues holds the extracted configuration values +type ConfigValues struct { + DashboardDomain string + LetsEncryptEmail string + BadgerVersion string +} + +// ReadTraefikConfig reads and extracts values from Traefik configuration files +func ReadTraefikConfig(mainConfigPath, dynamicConfigPath string) (*ConfigValues, error) { + // Read main config file + mainConfigData, err := os.ReadFile(mainConfigPath) + if err != nil { + return nil, fmt.Errorf("error reading main config file: %w", err) + } + + var mainConfig TraefikConfig + if err := yaml.Unmarshal(mainConfigData, &mainConfig); err != nil { + return nil, fmt.Errorf("error parsing main config file: %w", err) + } + + // Read dynamic config file + dynamicConfigData, err := os.ReadFile(dynamicConfigPath) + if err != nil { + return nil, fmt.Errorf("error reading dynamic config file: %w", err) + } + + var dynamicConfig DynamicConfig + if err := yaml.Unmarshal(dynamicConfigData, &dynamicConfig); err != nil { + return nil, fmt.Errorf("error parsing dynamic config file: %w", err) + } + + // Extract values + values := &ConfigValues{ + BadgerVersion: mainConfig.Experimental.Plugins.Badger.Version, + LetsEncryptEmail: mainConfig.CertificatesResolvers.LetsEncrypt.Acme.Email, + } + + // Extract DashboardDomain from router rules + // Look for it in the main router rules + for _, router := range dynamicConfig.HTTP.Routers { + if router.Rule != "" { + // Extract domain from Host(`mydomain.com`) + if domain := extractDomainFromRule(router.Rule); domain != "" { + values.DashboardDomain = domain + break + } + } + } + + return values, nil +} + +// extractDomainFromRule extracts the domain from a router rule +func extractDomainFromRule(rule string) string { + // Look for the Host(`mydomain.com`) pattern + if start := findPattern(rule, "Host(`"); start != -1 { + end := findPattern(rule[start:], "`)") + if end != -1 { + return rule[start+6 : start+end] + } + } + return "" +} + +// findPattern finds the start of a pattern in a string +func findPattern(s, pattern string) int { + return bytes.Index([]byte(s), []byte(pattern)) +} + +type Volume string +type Port string +type Expose string + +type HealthCheck struct { + Test []string `yaml:"test,omitempty"` + Interval string `yaml:"interval,omitempty"` + Timeout string `yaml:"timeout,omitempty"` + Retries int `yaml:"retries,omitempty"` +} + +type DependsOnCondition struct { + Condition string `yaml:"condition,omitempty"` +} + +type Service struct { + Image string `yaml:"image,omitempty"` + ContainerName string `yaml:"container_name,omitempty"` + Environment map[string]string `yaml:"environment,omitempty"` + HealthCheck *HealthCheck `yaml:"healthcheck,omitempty"` + DependsOn map[string]DependsOnCondition `yaml:"depends_on,omitempty"` + Labels []string `yaml:"labels,omitempty"` + Volumes []Volume `yaml:"volumes,omitempty"` + Ports []Port `yaml:"ports,omitempty"` + Expose []Expose `yaml:"expose,omitempty"` + Restart string `yaml:"restart,omitempty"` + Command interface{} `yaml:"command,omitempty"` + NetworkMode string `yaml:"network_mode,omitempty"` + CapAdd []string `yaml:"cap_add,omitempty"` +} + +type Network struct { + Driver string `yaml:"driver,omitempty"` + Name string `yaml:"name,omitempty"` +} + +type DockerConfig struct { + Version string `yaml:"version,omitempty"` + Services map[string]Service `yaml:"services"` + Networks map[string]Network `yaml:"networks,omitempty"` +} + +func AddCrowdSecService(configPath string) error { + // Read existing config + data, err := os.ReadFile(configPath) + if err != nil { + return err + } + + // Parse existing config + var config DockerConfig + if err := yaml.Unmarshal(data, &config); err != nil { + return err + } + + // Create CrowdSec service + crowdsecService := Service{ + Image: "crowdsecurity/crowdsec:latest", + ContainerName: "crowdsec", + Environment: map[string]string{ + "GID": "${GID-1000}", + "COLLECTIONS": "crowdsecurity/traefik crowdsecurity/appsec-virtual-patching crowdsecurity/appsec-generic-rules", + "ENROLL_INSTANCE_NAME": "pangolin-crowdsec", + "PARSERS": "crowdsecurity/whitelists", + "ACQUIRE_FILES": "/var/log/traefik/*.log", + "ENROLL_TAGS": "docker", + }, + HealthCheck: &HealthCheck{ + Test: []string{"CMD", "cscli", "capi", "status"}, + }, + DependsOn: map[string]DependsOnCondition{ + "gerbil": {}, + }, + Labels: []string{"traefik.enable=false"}, + Volumes: []Volume{ + "./config/crowdsec:/etc/crowdsec", + "./config/crowdsec/db:/var/lib/crowdsec/data", + "./config/crowdsec_logs/auth.log:/var/log/auth.log:ro", + "./config/crowdsec_logs/syslog:/var/log/syslog:ro", + "./config/crowdsec_logs:/var/log", + "./config/traefik/logs:/var/log/traefik", + }, + Ports: []Port{ + "9090:9090", + "6060:6060", + }, + Expose: []Expose{ + "9090", + "6060", + "7422", + }, + Restart: "unless-stopped", + Command: "-t", + } + + // Add CrowdSec service to config + if config.Services == nil { + config.Services = make(map[string]Service) + } + config.Services["crowdsec"] = crowdsecService + + // Marshal config with better formatting + yamlData, err := yaml.Marshal(&config) + if err != nil { + return err + } + + // Write config back to file + return os.WriteFile(configPath, yamlData, 0644) +} diff --git a/install/fs/config.yml b/install/config/config.yml similarity index 100% rename from install/fs/config.yml rename to install/config/config.yml diff --git a/install/crowdsec/acquis.yaml b/install/config/crowdsec/acquis.yaml similarity index 100% rename from install/crowdsec/acquis.yaml rename to install/config/crowdsec/acquis.yaml diff --git a/install/crowdsec/config.yaml b/install/config/crowdsec/config.yaml similarity index 100% rename from install/crowdsec/config.yaml rename to install/config/crowdsec/config.yaml diff --git a/install/crowdsec/dynamic_config.yml b/install/config/crowdsec/dynamic_config.yml similarity index 92% rename from install/crowdsec/dynamic_config.yml rename to install/config/crowdsec/dynamic_config.yml index 9175b143..d2556971 100644 --- a/install/crowdsec/dynamic_config.yml +++ b/install/config/crowdsec/dynamic_config.yml @@ -56,7 +56,7 @@ http: routers: # HTTP to HTTPS redirect router main-app-router-redirect: - rule: "Host(`{{.DomainName}}`)" # Dynamic Domain Name + rule: "Host(`{{.DashboardDomain}}`)" # Dynamic Domain Name service: next-service entryPoints: - web @@ -65,7 +65,7 @@ http: # Next.js router (handles everything except API and WebSocket paths) next-router: - rule: "Host(`{{.DomainName}}`) && !PathPrefix(`/api/v1`)" # Dynamic Domain Name + rule: "Host(`{{.DashboardDomain}}`) && !PathPrefix(`/api/v1`)" # Dynamic Domain Name service: next-service entryPoints: - websecure @@ -76,7 +76,7 @@ http: # API router (handles /api/v1 paths) api-router: - rule: "Host(`{{.DomainName}}`) && PathPrefix(`/api/v1`)" # Dynamic Domain Name + rule: "Host(`{{.DashboardDomain}}`) && PathPrefix(`/api/v1`)" # Dynamic Domain Name service: api-service entryPoints: - websecure @@ -87,7 +87,7 @@ http: # WebSocket router ws-router: - rule: "Host(`{{.DomainName}}`)" # Dynamic Domain Name + rule: "Host(`{{.DashboardDomain}}`)" # Dynamic Domain Name service: api-service entryPoints: - websecure diff --git a/install/crowdsec/local_api_credentials.yaml b/install/config/crowdsec/local_api_credentials.yaml similarity index 100% rename from install/crowdsec/local_api_credentials.yaml rename to install/config/crowdsec/local_api_credentials.yaml diff --git a/install/crowdsec/profiles.yaml b/install/config/crowdsec/profiles.yaml similarity index 100% rename from install/crowdsec/profiles.yaml rename to install/config/crowdsec/profiles.yaml diff --git a/install/crowdsec/traefik_config.yml b/install/config/crowdsec/traefik_config.yml similarity index 100% rename from install/crowdsec/traefik_config.yml rename to install/config/crowdsec/traefik_config.yml diff --git a/install/fs/docker-compose.yml b/install/config/docker-compose.yml similarity index 97% rename from install/fs/docker-compose.yml rename to install/config/docker-compose.yml index ea673eb0..42604ab4 100644 --- a/install/fs/docker-compose.yml +++ b/install/config/docker-compose.yml @@ -10,7 +10,6 @@ services: interval: "3s" timeout: "3s" retries: 5 - {{if .InstallGerbil}} gerbil: image: fosrl/gerbil:{{.GerbilVersion}} @@ -34,15 +33,13 @@ services: - 443:443 # Port for traefik because of the network_mode - 80:80 # Port for traefik because of the network_mode {{end}} - traefik: image: traefik:v3.3.3 container_name: traefik restart: unless-stopped {{if .InstallGerbil}} network_mode: service:gerbil # Ports appear on the gerbil service -{{end}} -{{if not .InstallGerbil}} +{{end}}{{if not .InstallGerbil}} ports: - 443:443 - 80:80 diff --git a/install/fs/traefik/dynamic_config.yml b/install/config/traefik/dynamic_config.yml similarity index 100% rename from install/fs/traefik/dynamic_config.yml rename to install/config/traefik/dynamic_config.yml diff --git a/install/fs/traefik/traefik_config.yml b/install/config/traefik/traefik_config.yml similarity index 100% rename from install/fs/traefik/traefik_config.yml rename to install/config/traefik/traefik_config.yml diff --git a/install/crowdsec.go b/install/crowdsec.go index 187fea88..02e9c71b 100644 --- a/install/crowdsec.go +++ b/install/crowdsec.go @@ -2,45 +2,26 @@ package main import ( "bytes" - "embed" "fmt" - "html/template" - "io" - "io/fs" "os" "os/exec" - "path/filepath" "strings" "time" ) -//go:embed crowdsec/* -var configCrowdsecFiles embed.FS - -// DockerContainer represents a Docker container -type DockerContainer struct { - NetworkSettings struct { - Networks map[string]struct { - IPAddress string `json:"IPAddress"` - } `json:"Networks"` - } `json:"NetworkSettings"` -} - -func installCrowdsec() error { - // Create configuration - config := &Config{} - +func installCrowdsec(config Config) error { // Run installation steps if err := backupConfig(); err != nil { return fmt.Errorf("backup failed: %v", err) } - if err := modifyDockerCompose(); err != nil { - return fmt.Errorf("docker-compose modification failed: %v", err) + if err := AddCrowdSecService("docker-compose.yml"); err != nil { + return fmt.Errorf("crowdsec service addition failed: %v", err) } - if err := createCrowdsecFiles(*config); err != nil { - return fmt.Errorf("config file creation failed: %v", err) + if err := createConfigFiles(config); err != nil { + fmt.Printf("Error creating config files: %v\n", err) + os.Exit(1) } moveFile("config/crowdsec/traefik_config.yaml", "config/traefik/traefik_config.yaml") @@ -50,14 +31,6 @@ func installCrowdsec() error { return fmt.Errorf("bouncer key retrieval failed: %v", err) } - if err := deployStack(); err != nil { - return fmt.Errorf("deployment failed: %v", err) - } - - if err := verifyDeployment(); err != nil { - return fmt.Errorf("verification failed: %v", err) - } - return nil } @@ -80,33 +53,7 @@ func backupConfig() error { return nil } -func modifyDockerCompose() error { - // Read existing docker-compose.yml - content, err := os.ReadFile("docker-compose.yml") - if err != nil { - return fmt.Errorf("failed to read docker-compose.yml: %v", err) - } - - // Verify required services exist - requiredServices := []string{"services:", "pangolin:", "gerbil:", "traefik:"} - for _, service := range requiredServices { - if !bytes.Contains(content, []byte(service)) { - return fmt.Errorf("required service %s not found in docker-compose.yml", service) - } - } - - // Add crowdsec service - modified := addCrowdsecService(string(content)) - - // Write modified content - if err := os.WriteFile("docker-compose.yml", []byte(modified), 0644); err != nil { - return fmt.Errorf("failed to write modified docker-compose.yml: %v", err) - } - - return nil -} - -func retrieveBouncerKey(config *Config) error { +func retrieveBouncerKey(config Config) error { // Start crowdsec container cmd := exec.Command("docker", "compose", "up", "-d", "crowdsec") if err := cmd.Run(); err != nil { @@ -114,8 +61,24 @@ func retrieveBouncerKey(config *Config) error { } defer exec.Command("docker", "compose", "down").Run() - // Wait for container to start - time.Sleep(10 * time.Second) + // verify that the container is running if not keep waiting for 10 more seconds then return an error + count := 0 + for { + cmd := exec.Command("docker", "inspect", "-f", "{{.State.Running}}", "crowdsec") + output, err := cmd.Output() + if err != nil { + return fmt.Errorf("failed to inspect crowdsec container: %v", err) + } + if strings.TrimSpace(string(output)) == "true" { + break + } + time.Sleep(10 * time.Second) + count++ + + if count > 4 { + return fmt.Errorf("crowdsec container is not running") + } + } // Get bouncer key output, err := exec.Command("docker", "exec", "crowdsec", "cscli", "bouncers", "add", "traefik-bouncer").Output() @@ -135,181 +98,13 @@ func retrieveBouncerKey(config *Config) error { return nil } -func deployStack() error { - cmd := exec.Command("docker", "compose", "up", "-d") - if err := cmd.Run(); err != nil { - return fmt.Errorf("failed to deploy stack: %v", err) - } - - fmt.Println("Stack deployed. Waiting 2 minutes for services to initialize...") - time.Sleep(2 * time.Minute) - return nil -} - -func verifyDeployment() error { - resp, err := exec.Command("curl", "-s", "http://localhost:6060/metrics").Output() +func checkIsCrowdsecInstalledInCompose() bool { + // Read docker-compose.yml + content, err := os.ReadFile("docker-compose.yml") if err != nil { - return fmt.Errorf("failed to get metrics: %v", err) + return false } - if !bytes.Contains(resp, []byte("appsec")) { - return fmt.Errorf("appsec metrics not found in response") - } - - return nil -} - -func copyFile(src, dst string) error { - source, err := os.Open(src) - if err != nil { - return err - } - defer source.Close() - - destination, err := os.Create(dst) - if err != nil { - return err - } - defer destination.Close() - - _, err = io.Copy(destination, source) - return err -} - -func moveFile(src, dst string) error { - if err := copyFile(src, dst); err != nil { - return err - } - - return os.Remove(src) -} - -func addCrowdsecService(content string) string { - // Implementation of adding crowdsec service to docker-compose.yml - // This would involve string manipulation or template rendering - // The actual implementation would depend on how you want to structure the docker-compose modifications - return content + ` - crowdsec: - image: crowdsecurity/crowdsec:latest - container_name: crowdsec - environment: - GID: "${GID-1000}" - COLLECTIONS: crowdsecurity/traefik crowdsecurity/appsec-virtual-patching crowdsecurity/appsec-generic-rules - ENROLL_INSTANCE_NAME: "pangolin-crowdsec" - PARSERS: crowdsecurity/whitelists - ACQUIRE_FILES: "/var/log/traefik/*.log" - ENROLL_TAGS: docker - healthcheck: - test: ["CMD", "cscli", "capi", "status"] - depends_on: - - gerbil - labels: - - "traefik.enable=false" - volumes: - - ./config/crowdsec:/etc/crowdsec - - ./config/crowdsec/db:/var/lib/crowdsec/data - - ./config/crowdsec_logs/auth.log:/var/log/auth.log:ro - - ./config/crowdsec_logs/syslog:/var/log/syslog:ro - - ./config/crowdsec_logs:/var/log - - ./config/traefik/logs:/var/log/traefik - ports: - - 9090:9090 - - 6060:6060 - expose: - - 9090 - - 6060 - - 7422 - restart: unless-stopped - command: -t` -} - -func createCrowdsecFiles(config Config) error { - // Walk through all embedded files - err := fs.WalkDir(configCrowdsecFiles, "crowdsec", func(path string, d fs.DirEntry, err error) error { - if err != nil { - return err - } - - // Skip the root fs directory itself - if path == "fs" { - return nil - } - - // Get the relative path by removing the "fs/" prefix - relPath := strings.TrimPrefix(path, "fs/") - - // skip .DS_Store - if strings.Contains(relPath, ".DS_Store") { - return nil - } - - // Create the full output path under "config/" - outPath := filepath.Join("config", relPath) - - if d.IsDir() { - // Create directory - if err := os.MkdirAll(outPath, 0755); err != nil { - return fmt.Errorf("failed to create directory %s: %v", outPath, err) - } - return nil - } - - // Read the template file - content, err := configFiles.ReadFile(path) - if err != nil { - return fmt.Errorf("failed to read %s: %v", path, err) - } - - // Parse template - tmpl, err := template.New(d.Name()).Parse(string(content)) - if err != nil { - return fmt.Errorf("failed to parse template %s: %v", path, err) - } - - // Ensure parent directory exists - if err := os.MkdirAll(filepath.Dir(outPath), 0755); err != nil { - return fmt.Errorf("failed to create parent directory for %s: %v", outPath, err) - } - - // Create output file - outFile, err := os.Create(outPath) - if err != nil { - return fmt.Errorf("failed to create %s: %v", outPath, err) - } - defer outFile.Close() - - // Execute template - if err := tmpl.Execute(outFile, config); err != nil { - return fmt.Errorf("failed to execute template %s: %v", path, err) - } - - return nil - }) - - if err != nil { - return fmt.Errorf("error walking config files: %v", err) - } - - // get the current directory - dir, err := os.Getwd() - if err != nil { - return fmt.Errorf("failed to get current directory: %v", err) - } - - sourcePath := filepath.Join(dir, "config/docker-compose.yml") - destPath := filepath.Join(dir, "docker-compose.yml") - - // Check if source file exists - if _, err := os.Stat(sourcePath); err != nil { - return fmt.Errorf("source docker-compose.yml not found: %v", err) - } - - // Try to move the file - err = os.Rename(sourcePath, destPath) - if err != nil { - return fmt.Errorf("failed to move docker-compose.yml from %s to %s: %v", - sourcePath, destPath, err) - } - - return nil + // Check for crowdsec service + return bytes.Contains(content, []byte("crowdsec:")) } diff --git a/install/go.mod b/install/go.mod index 85cf49e4..536ac2dd 100644 --- a/install/go.mod +++ b/install/go.mod @@ -5,4 +5,5 @@ go 1.23.0 require ( golang.org/x/sys v0.29.0 // indirect golang.org/x/term v0.28.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/install/go.sum b/install/go.sum index f05f63b4..3316e039 100644 --- a/install/go.sum +++ b/install/go.sum @@ -2,3 +2,6 @@ golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.28.0 h1:/Ts8HFuMR2E6IP/jlo7QVLZHggjKQbhu/7H0LJFr3Gg= golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/install/main.go b/install/main.go index d5bf5e1e..4f7c7df6 100644 --- a/install/main.go +++ b/install/main.go @@ -6,6 +6,7 @@ import ( "fmt" "io/fs" "os" + "io" "os/exec" "path/filepath" "runtime" @@ -24,7 +25,7 @@ func loadVersions(config *Config) { config.BadgerVersion = "replaceme" } -//go:embed fs/* +//go:embed config/* var configFiles embed.FS type Config struct { @@ -46,6 +47,7 @@ type Config struct { EmailNoReply string InstallGerbil bool TraefikBouncerKey string + DoCrowdsecInstall bool } func main() { @@ -57,9 +59,12 @@ func main() { os.Exit(1) } + var config Config + config.DoCrowdsecInstall = false + // check if there is already a config file if _, err := os.Stat("config/config.yml"); err != nil { - config := collectUserInput(reader) + config = collectUserInput(reader) loadVersions(&config) @@ -68,18 +73,53 @@ func main() { os.Exit(1) } + moveFile("config/docker-compose.yml", "docker-compose.yml") + if !isDockerInstalled() && runtime.GOOS == "linux" { if readBool(reader, "Docker is not installed. Would you like to install it?", true) { installDocker() } } + + fmt.Println("\n=== Starting installation ===") + + if isDockerInstalled() { + if readBool(reader, "Would you like to install and start the containers?", true) { + pullAndStartContainers() + } + } } else { - fmt.Println("Config file already exists... skipping configuration") + fmt.Println("Looks like you already installed, so I am going to do the setup...") } - if isDockerInstalled() { - if readBool(reader, "Would you like to install and start the containers?", true) { - pullAndStartContainers() + if !checkIsCrowdsecInstalledInCompose() { + fmt.Println("\n=== Crowdsec Install ===") + // check if crowdsec is installed + if readBool(reader, "Would you like to install Crowdsec?", true) { + + if config.DashboardDomain == "" { + traefikConfig, err := ReadTraefikConfig("config/traefik/traefik_config.yml", "config/traefik/dynamic_config.yml") + if err != nil { + fmt.Printf("Error reading config: %v\n", err) + return + } + config.DashboardDomain = traefikConfig.DashboardDomain + config.LetsEncryptEmail = traefikConfig.LetsEncryptEmail + config.BadgerVersion = traefikConfig.BadgerVersion + + // print the values and check if they are right + fmt.Println("Detected values:") + fmt.Printf("Dashboard Domain: %s\n", config.DashboardDomain) + fmt.Printf("Let's Encrypt Email: %s\n", config.LetsEncryptEmail) + fmt.Printf("Badger Version: %s\n", config.BadgerVersion) + + if !readBool(reader, "Are these values correct?", true) { + config = collectUserInput(reader) + } + } + + config.DoCrowdsecInstall = true + installCrowdsec(config) } } @@ -137,6 +177,11 @@ func readInt(reader *bufio.Reader, prompt string, defaultValue int) int { return value } +func isDockerFilePresent() bool { + _, err := os.Stat("docker-compose.yml") + return !os.IsNotExist(err) +} + func collectUserInput(reader *bufio.Reader) Config { config := Config{} @@ -262,31 +307,33 @@ func createConfigFiles(config Config) error { os.MkdirAll("config/logs", 0755) // Walk through all embedded files - err := fs.WalkDir(configFiles, "fs", func(path string, d fs.DirEntry, err error) error { + err := fs.WalkDir(configFiles, "config", func(path string, d fs.DirEntry, err error) error { if err != nil { return err } // Skip the root fs directory itself - if path == "fs" { + if path == "config" { return nil } - // Get the relative path by removing the "fs/" prefix - relPath := strings.TrimPrefix(path, "fs/") + if !config.DoCrowdsecInstall && strings.Contains(path, "crowdsec") { + return nil + } + + if config.DoCrowdsecInstall && !strings.Contains(path, "crowdsec") { + return nil + } // skip .DS_Store - if strings.Contains(relPath, ".DS_Store") { + if strings.Contains(path, ".DS_Store") { return nil } - // Create the full output path under "config/" - outPath := filepath.Join("config", relPath) - if d.IsDir() { // Create directory - if err := os.MkdirAll(outPath, 0755); err != nil { - return fmt.Errorf("failed to create directory %s: %v", outPath, err) + if err := os.MkdirAll(path, 0755); err != nil { + return fmt.Errorf("failed to create directory %s: %v", path, err) } return nil } @@ -304,14 +351,14 @@ func createConfigFiles(config Config) error { } // Ensure parent directory exists - if err := os.MkdirAll(filepath.Dir(outPath), 0755); err != nil { - return fmt.Errorf("failed to create parent directory for %s: %v", outPath, err) + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + return fmt.Errorf("failed to create parent directory for %s: %v", path, err) } // Create output file - outFile, err := os.Create(outPath) + outFile, err := os.Create(path) if err != nil { - return fmt.Errorf("failed to create %s: %v", outPath, err) + return fmt.Errorf("failed to create %s: %v", path, err) } defer outFile.Close() @@ -327,30 +374,10 @@ func createConfigFiles(config Config) error { return fmt.Errorf("error walking config files: %v", err) } - // get the current directory - dir, err := os.Getwd() - if err != nil { - return fmt.Errorf("failed to get current directory: %v", err) - } - - sourcePath := filepath.Join(dir, "config/docker-compose.yml") - destPath := filepath.Join(dir, "docker-compose.yml") - - // Check if source file exists - if _, err := os.Stat(sourcePath); err != nil { - return fmt.Errorf("source docker-compose.yml not found: %v", err) - } - - // Try to move the file - err = os.Rename(sourcePath, destPath) - if err != nil { - return fmt.Errorf("failed to move docker-compose.yml from %s to %s: %v", - sourcePath, destPath, err) - } - return nil } + func installDocker() error { // Detect Linux distribution cmd := exec.Command("cat", "/etc/os-release") @@ -491,3 +518,28 @@ func pullAndStartContainers() error { return nil } + +func copyFile(src, dst string) error { + source, err := os.Open(src) + if err != nil { + return err + } + defer source.Close() + + destination, err := os.Create(dst) + if err != nil { + return err + } + defer destination.Close() + + _, err = io.Copy(destination, source) + return err +} + +func moveFile(src, dst string) error { + if err := copyFile(src, dst); err != nil { + return err + } + + return os.Remove(src) +} \ No newline at end of file From f61d442989fc758f4b5cfbbc5ee07e02962290d7 Mon Sep 17 00:00:00 2001 From: Owen Date: Fri, 14 Feb 2025 09:51:17 -0500 Subject: [PATCH 05/26] Allow . in path; resolves #199 --- server/lib/validators.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/lib/validators.ts b/server/lib/validators.ts index 0aa590e6..f4be6277 100644 --- a/server/lib/validators.ts +++ b/server/lib/validators.ts @@ -35,7 +35,7 @@ export function isValidUrlGlobPattern(pattern: string): boolean { } // Check for invalid characters - if (!/^[a-zA-Z0-9_*-]*$/.test(segment)) { + if (!/^[a-zA-Z0-9_.*-]*$/.test(segment)) { return false; } } From 4c1366ef91ce101b0c5aaff33d0b44f50942b642 Mon Sep 17 00:00:00 2001 From: Milo Schwartz Date: Fri, 14 Feb 2025 12:27:03 -0500 Subject: [PATCH 06/26] force router refresh on save closes #198 --- .../settings/resources/[resourceId]/authentication/page.tsx | 6 ++++++ .../settings/resources/[resourceId]/connectivity/page.tsx | 4 ++++ .../[orgId]/settings/resources/[resourceId]/rules/page.tsx | 6 +++++- 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/app/[orgId]/settings/resources/[resourceId]/authentication/page.tsx b/src/app/[orgId]/settings/resources/[resourceId]/authentication/page.tsx index 0e3dc7bc..df5376f9 100644 --- a/src/app/[orgId]/settings/resources/[resourceId]/authentication/page.tsx +++ b/src/app/[orgId]/settings/resources/[resourceId]/authentication/page.tsx @@ -49,6 +49,7 @@ import { } from "@app/components/Settings"; import { SwitchInput } from "@app/components/SwitchInput"; import { InfoPopup } from "@app/components/ui/info-popup"; +import { useRouter } from "next/navigation"; const UsersRolesFormSchema = z.object({ roles: z.array( @@ -82,6 +83,7 @@ export default function ResourceAuthenticationPage() { const { env } = useEnvContext(); const api = createApiClient({ env }); + const router = useRouter(); const [pageLoading, setPageLoading] = useState(true); @@ -236,6 +238,7 @@ export default function ResourceAuthenticationPage() { title: "Saved successfully", description: "Whitelist settings have been saved" }); + router.refresh(); } catch (e) { console.error(e); toast({ @@ -283,6 +286,7 @@ export default function ResourceAuthenticationPage() { title: "Saved successfully", description: "Authentication settings have been saved" }); + router.refresh(); } catch (e) { console.error(e); toast({ @@ -314,6 +318,7 @@ export default function ResourceAuthenticationPage() { updateAuthInfo({ password: false }); + router.refresh(); }) .catch((e) => { toast({ @@ -344,6 +349,7 @@ export default function ResourceAuthenticationPage() { updateAuthInfo({ pincode: false }); + router.refresh(); }) .catch((e) => { toast({ diff --git a/src/app/[orgId]/settings/resources/[resourceId]/connectivity/page.tsx b/src/app/[orgId]/settings/resources/[resourceId]/connectivity/page.tsx index dfd2f66c..ea67e23e 100644 --- a/src/app/[orgId]/settings/resources/[resourceId]/connectivity/page.tsx +++ b/src/app/[orgId]/settings/resources/[resourceId]/connectivity/page.tsx @@ -64,6 +64,7 @@ import { import { SwitchInput } from "@app/components/SwitchInput"; import { useSiteContext } from "@app/hooks/useSiteContext"; import { InfoPopup } from "@app/components/ui/info-popup"; +import { useRouter } from "next/navigation"; // Regular expressions for validation const DOMAIN_REGEX = @@ -125,6 +126,7 @@ export default function ReverseProxyTargets(props: { const [loading, setLoading] = useState(false); const [pageLoading, setPageLoading] = useState(true); + const router = useRouter(); const addTargetForm = useForm({ resolver: zodResolver(addTargetSchema), @@ -299,6 +301,7 @@ export default function ReverseProxyTargets(props: { }); setTargetsToRemove([]); + router.refresh(); } catch (err) { console.error(err); toast({ @@ -339,6 +342,7 @@ export default function ReverseProxyTargets(props: { title: "SSL Configuration", description: "SSL configuration updated successfully" }); + router.refresh(); } } diff --git a/src/app/[orgId]/settings/resources/[resourceId]/rules/page.tsx b/src/app/[orgId]/settings/resources/[resourceId]/rules/page.tsx index 7fc16b81..1b9eb6ca 100644 --- a/src/app/[orgId]/settings/resources/[resourceId]/rules/page.tsx +++ b/src/app/[orgId]/settings/resources/[resourceId]/rules/page.tsx @@ -71,6 +71,7 @@ import { isValidUrlGlobPattern } from "@server/lib/validators"; import { Switch } from "@app/components/ui/switch"; +import { useRouter } from "next/navigation"; // Schema for rule validation const addRuleSchema = z.object({ @@ -107,6 +108,7 @@ export default function ResourceRules(props: { const [loading, setLoading] = useState(false); const [pageLoading, setPageLoading] = useState(true); const [rulesEnabled, setRulesEnabled] = useState(resource.applyRules); + const router = useRouter(); const addRuleForm = useForm({ resolver: zodResolver(addRuleSchema), @@ -253,6 +255,7 @@ export default function ResourceRules(props: { title: "Enable Rules", description: "Rule evaluation has been updated" }); + router.refresh(); } } @@ -370,6 +373,7 @@ export default function ResourceRules(props: { }); setRulesToRemove([]); + router.refresh(); } catch (err) { console.error(err); toast({ @@ -590,7 +594,7 @@ export default function ResourceRules(props: { { await saveApplyRules(val); }} From 40922fedb8b3ad87bd1fb10aa89a85a45907ea8e Mon Sep 17 00:00:00 2001 From: Owen Date: Fri, 14 Feb 2025 12:32:10 -0500 Subject: [PATCH 07/26] Support v6 --- server/lib/ip.ts | 145 ++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 124 insertions(+), 21 deletions(-) diff --git a/server/lib/ip.ts b/server/lib/ip.ts index 88c64acc..62cf1d2d 100644 --- a/server/lib/ip.ts +++ b/server/lib/ip.ts @@ -3,24 +3,98 @@ interface IPRange { end: bigint; } +type IPVersion = 4 | 6; + /** - * Converts IP address string to BigInt for numerical operations + * Detects IP version from address string + */ +function detectIpVersion(ip: string): IPVersion { + return ip.includes(':') ? 6 : 4; +} + +/** + * Converts IPv4 or IPv6 address string to BigInt for numerical operations */ function ipToBigInt(ip: string): bigint { - return ip.split('.') - .reduce((acc, octet) => BigInt.asUintN(64, (acc << BigInt(8)) + BigInt(parseInt(octet))), BigInt(0)); + const version = detectIpVersion(ip); + + if (version === 4) { + return ip.split('.') + .reduce((acc, octet) => { + const num = parseInt(octet); + if (isNaN(num) || num < 0 || num > 255) { + throw new Error(`Invalid IPv4 octet: ${octet}`); + } + return BigInt.asUintN(64, (acc << BigInt(8)) + BigInt(num)); + }, BigInt(0)); + } else { + // Handle IPv6 + // Expand :: notation + let fullAddress = ip; + if (ip.includes('::')) { + const parts = ip.split('::'); + if (parts.length > 2) throw new Error('Invalid IPv6 address: multiple :: found'); + const missing = 8 - (parts[0].split(':').length + parts[1].split(':').length); + const padding = Array(missing).fill('0').join(':'); + fullAddress = `${parts[0]}:${padding}:${parts[1]}`; + } + + return fullAddress.split(':') + .reduce((acc, hextet) => { + const num = parseInt(hextet || '0', 16); + if (isNaN(num) || num < 0 || num > 65535) { + throw new Error(`Invalid IPv6 hextet: ${hextet}`); + } + return BigInt.asUintN(128, (acc << BigInt(16)) + BigInt(num)); + }, BigInt(0)); + } } /** * Converts BigInt to IP address string */ -function bigIntToIp(num: bigint): string { - const octets: number[] = []; - for (let i = 0; i < 4; i++) { - octets.unshift(Number(num & BigInt(255))); - num = num >> BigInt(8); +function bigIntToIp(num: bigint, version: IPVersion): string { + if (version === 4) { + const octets: number[] = []; + for (let i = 0; i < 4; i++) { + octets.unshift(Number(num & BigInt(255))); + num = num >> BigInt(8); + } + return octets.join('.'); + } else { + const hextets: string[] = []; + for (let i = 0; i < 8; i++) { + hextets.unshift(Number(num & BigInt(65535)).toString(16).padStart(4, '0')); + num = num >> BigInt(16); + } + // Compress zero sequences + let maxZeroStart = -1; + let maxZeroLength = 0; + let currentZeroStart = -1; + let currentZeroLength = 0; + + for (let i = 0; i < hextets.length; i++) { + if (hextets[i] === '0000') { + if (currentZeroStart === -1) currentZeroStart = i; + currentZeroLength++; + if (currentZeroLength > maxZeroLength) { + maxZeroLength = currentZeroLength; + maxZeroStart = currentZeroStart; + } + } else { + currentZeroStart = -1; + currentZeroLength = 0; + } + } + + if (maxZeroLength > 1) { + hextets.splice(maxZeroStart, maxZeroLength, ''); + if (maxZeroStart === 0) hextets.unshift(''); + if (maxZeroStart + maxZeroLength === 8) hextets.push(''); + } + + return hextets.map(h => h === '0000' ? '0' : h.replace(/^0+/, '')).join(':'); } - return octets.join('.'); } /** @@ -28,33 +102,56 @@ function bigIntToIp(num: bigint): string { */ function cidrToRange(cidr: string): IPRange { const [ip, prefix] = cidr.split('/'); + const version = detectIpVersion(ip); const prefixBits = parseInt(prefix); const ipBigInt = ipToBigInt(ip); - const mask = BigInt.asUintN(64, (BigInt(1) << BigInt(32 - prefixBits)) - BigInt(1)); + + // Validate prefix length + const maxPrefix = version === 4 ? 32 : 128; + if (prefixBits < 0 || prefixBits > maxPrefix) { + throw new Error(`Invalid prefix length for IPv${version}: ${prefix}`); + } + + const shiftBits = BigInt(maxPrefix - prefixBits); + const mask = BigInt.asUintN(version === 4 ? 64 : 128, (BigInt(1) << shiftBits) - BigInt(1)); const start = ipBigInt & ~mask; const end = start | mask; + return { start, end }; } /** * Finds the next available CIDR block given existing allocations * @param existingCidrs Array of existing CIDR blocks - * @param blockSize Desired prefix length for the new block (e.g., 24 for /24) - * @param startCidr Optional CIDR to start searching from (default: "0.0.0.0/0") + * @param blockSize Desired prefix length for the new block + * @param startCidr Optional CIDR to start searching from * @returns Next available CIDR block or null if none found */ export function findNextAvailableCidr( existingCidrs: string[], blockSize: number, - startCidr: string = "0.0.0.0/0" + startCidr?: string ): string | null { + if (existingCidrs.length === 0) return null; + + // Determine IP version from first CIDR + const version = detectIpVersion(existingCidrs[0].split('/')[0]); + // Use appropriate default startCidr if none provided + startCidr = startCidr || (version === 4 ? "0.0.0.0/0" : "::/0"); + + // Ensure all CIDRs are same version + if (existingCidrs.some(cidr => detectIpVersion(cidr.split('/')[0]) !== version)) { + throw new Error('All CIDRs must be of the same IP version'); + } + // Convert existing CIDRs to ranges and sort them const existingRanges = existingCidrs .map(cidr => cidrToRange(cidr)) .sort((a, b) => (a.start < b.start ? -1 : 1)); // Calculate block size - const blockSizeBigInt = BigInt(1) << BigInt(32 - blockSize); + const maxPrefix = version === 4 ? 32 : 128; + const blockSizeBigInt = BigInt(1) << BigInt(maxPrefix - blockSize); // Start from the beginning of the given CIDR let current = cidrToRange(startCidr).start; @@ -63,7 +160,6 @@ export function findNextAvailableCidr( // Iterate through existing ranges for (let i = 0; i <= existingRanges.length; i++) { const nextRange = existingRanges[i]; - // Align current to block size const alignedCurrent = current + ((blockSizeBigInt - (current % blockSizeBigInt)) % blockSizeBigInt); @@ -74,7 +170,7 @@ export function findNextAvailableCidr( // If we're at the end of existing ranges or found a gap if (!nextRange || alignedCurrent + blockSizeBigInt - BigInt(1) < nextRange.start) { - return `${bigIntToIp(alignedCurrent)}/${blockSize}`; + return `${bigIntToIp(alignedCurrent, version)}/${blockSize}`; } // Move current pointer to after the current range @@ -85,12 +181,19 @@ export function findNextAvailableCidr( } /** -* Checks if a given IP address is within a CIDR range -* @param ip IP address to check -* @param cidr CIDR range to check against -* @returns boolean indicating if IP is within the CIDR range -*/ + * Checks if a given IP address is within a CIDR range + * @param ip IP address to check + * @param cidr CIDR range to check against + * @returns boolean indicating if IP is within the CIDR range + */ export function isIpInCidr(ip: string, cidr: string): boolean { + const ipVersion = detectIpVersion(ip); + const cidrVersion = detectIpVersion(cidr.split('/')[0]); + + if (ipVersion !== cidrVersion) { + throw new Error('IP address and CIDR must be of the same version'); + } + const ipBigInt = ipToBigInt(ip); const range = cidrToRange(cidr); return ipBigInt >= range.start && ipBigInt <= range.end; From 7797c6c770b5e50d10f6eaf240a904a4ec1d58a7 Mon Sep 17 00:00:00 2001 From: Owen Date: Fri, 14 Feb 2025 12:38:28 -0500 Subject: [PATCH 08/26] Allow the chars from RFC 3986 --- server/lib/validators.ts | 37 ++++++++++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/server/lib/validators.ts b/server/lib/validators.ts index f4be6277..675c0809 100644 --- a/server/lib/validators.ts +++ b/server/lib/validators.ts @@ -11,7 +11,7 @@ export function isValidIP(ip: string): boolean { export function isValidUrlGlobPattern(pattern: string): boolean { // Remove leading slash if present pattern = pattern.startsWith("/") ? pattern.slice(1) : pattern; - + // Empty string is not valid if (!pattern) { return false; @@ -19,12 +19,12 @@ export function isValidUrlGlobPattern(pattern: string): boolean { // Split path into segments const segments = pattern.split("/"); - + // Check each segment for (let i = 0; i < segments.length; i++) { const segment = segments[i]; - - // Empty segments are not allowed (double slashes) + + // Empty segments are not allowed (double slashes), except at the end if (!segment && i !== segments.length - 1) { return false; } @@ -34,11 +34,30 @@ export function isValidUrlGlobPattern(pattern: string): boolean { return false; } - // Check for invalid characters - if (!/^[a-zA-Z0-9_.*-]*$/.test(segment)) { - return false; + // Check each character in the segment + for (let j = 0; j < segment.length; j++) { + const char = segment[j]; + + // Check for percent-encoded sequences + if (char === "%" && j + 2 < segment.length) { + const hex1 = segment[j + 1]; + const hex2 = segment[j + 2]; + if (!/^[0-9A-Fa-f]$/.test(hex1) || !/^[0-9A-Fa-f]$/.test(hex2)) { + return false; + } + j += 2; // Skip the next two characters + continue; + } + + // Allow: + // - unreserved (A-Z a-z 0-9 - . _ ~) + // - sub-delims (! $ & ' ( ) * + , ; =) + // - @ : for compatibility with some systems + if (!/^[A-Za-z0-9\-._~!$&'()*+,;=@:]$/.test(char)) { + return false; + } } } - + return true; -} +} \ No newline at end of file From 8dd30c88abeaaa31a5a2fe4bb66079dc57217995 Mon Sep 17 00:00:00 2001 From: Milo Schwartz Date: Fri, 14 Feb 2025 13:12:29 -0500 Subject: [PATCH 09/26] fix reset password sql error --- server/auth/sessions/app.ts | 32 ++++++++++++++++++++++++---- server/routers/auth/resetPassword.ts | 22 +++++++++++++------ 2 files changed, 43 insertions(+), 11 deletions(-) diff --git a/server/auth/sessions/app.ts b/server/auth/sessions/app.ts index 18ea072b..62850453 100644 --- a/server/auth/sessions/app.ts +++ b/server/auth/sessions/app.ts @@ -11,7 +11,7 @@ import { users } from "@server/db/schema"; import db from "@server/db"; -import { eq } from "drizzle-orm"; +import { eq, inArray } from "drizzle-orm"; import config from "@server/lib/config"; import type { RandomReader } from "@oslojs/crypto/random"; import { generateRandomString } from "@oslojs/crypto/random"; @@ -95,12 +95,36 @@ export async function validateSessionToken( } export async function invalidateSession(sessionId: string): Promise { - await db.delete(resourceSessions).where(eq(resourceSessions.userSessionId, sessionId)); - await db.delete(sessions).where(eq(sessions.sessionId, sessionId)); + try { + await db.transaction(async (trx) => { + await trx + .delete(resourceSessions) + .where(eq(resourceSessions.userSessionId, sessionId)); + await trx.delete(sessions).where(eq(sessions.sessionId, sessionId)); + }); + } catch (e) { + logger.error("Failed to invalidate session", e); + } } export async function invalidateAllSessions(userId: string): Promise { - await db.delete(sessions).where(eq(sessions.userId, userId)); + try { + await db.transaction(async (trx) => { + const userSessions = await trx + .select() + .from(sessions) + .where(eq(sessions.userId, userId)); + await trx.delete(resourceSessions).where( + inArray( + resourceSessions.userSessionId, + userSessions.map((s) => s.sessionId) + ) + ); + await trx.delete(sessions).where(eq(sessions.userId, userId)); + }); + } catch (e) { + logger.error("Failed to all invalidate user sessions", e); + } } export function serializeSessionCookie( diff --git a/server/routers/auth/resetPassword.ts b/server/routers/auth/resetPassword.ts index 97b283c6..ac1b6600 100644 --- a/server/routers/auth/resetPassword.ts +++ b/server/routers/auth/resetPassword.ts @@ -149,8 +149,6 @@ export async function resetPassword( const passwordHash = await hashPassword(newPassword); - await invalidateAllSessions(resetRequest[0].userId); - await db.transaction(async (trx) => { await trx .update(users) @@ -162,11 +160,21 @@ export async function resetPassword( .where(eq(passwordResetTokens.email, email)); }); - await sendEmail(ConfirmPasswordReset({ email }), { - from: config.getNoReplyEmail(), - to: email, - subject: "Password Reset Confirmation" - }); + try { + await invalidateAllSessions(resetRequest[0].userId); + } catch (e) { + logger.error("Failed to invalidate user sessions", e); + } + + try { + await sendEmail(ConfirmPasswordReset({ email }), { + from: config.getNoReplyEmail(), + to: email, + subject: "Password Reset Confirmation" + }); + } catch (e) { + logger.error("Failed to send password reset confirmation email", e); + } return response(res, { data: null, From 2ff6d1d117c408e9d0e747eed911deb2da34edbb Mon Sep 17 00:00:00 2001 From: Milo Schwartz Date: Fri, 14 Feb 2025 13:27:34 -0500 Subject: [PATCH 10/26] allow any string as target --- server/routers/target/createTarget.ts | 56 +++++++++---------- server/routers/target/updateTarget.ts | 56 +++++++++---------- .../[resourceId]/connectivity/page.tsx | 42 +++++++------- 3 files changed, 77 insertions(+), 77 deletions(-) diff --git a/server/routers/target/createTarget.ts b/server/routers/target/createTarget.ts index b1080d87..11f3de69 100644 --- a/server/routers/target/createTarget.ts +++ b/server/routers/target/createTarget.ts @@ -13,33 +13,33 @@ import { addTargets } from "../newt/targets"; import { eq } from "drizzle-orm"; import { pickPort } from "./helpers"; -// Regular expressions for validation -const DOMAIN_REGEX = - /^[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$/; -const IPV4_REGEX = - /^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$/; -const IPV6_REGEX = /^(?:[A-F0-9]{1,4}:){7}[A-F0-9]{1,4}$/i; - -// Schema for domain names and IP addresses -const domainSchema = z - .string() - .min(1, "Domain cannot be empty") - .max(255, "Domain name too long") - .refine( - (value) => { - // Check if it's a valid IP address (v4 or v6) - if (IPV4_REGEX.test(value) || IPV6_REGEX.test(value)) { - return true; - } - - // Check if it's a valid domain name - return DOMAIN_REGEX.test(value); - }, - { - message: "Invalid domain name or IP address format", - path: ["domain"] - } - ); +// // Regular expressions for validation +// const DOMAIN_REGEX = +// /^[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$/; +// const IPV4_REGEX = +// /^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$/; +// const IPV6_REGEX = /^(?:[A-F0-9]{1,4}:){7}[A-F0-9]{1,4}$/i; +// +// // Schema for domain names and IP addresses +// const domainSchema = z +// .string() +// .min(1, "Domain cannot be empty") +// .max(255, "Domain name too long") +// .refine( +// (value) => { +// // Check if it's a valid IP address (v4 or v6) +// if (IPV4_REGEX.test(value) || IPV6_REGEX.test(value)) { +// return true; +// } +// +// // Check if it's a valid domain name +// return DOMAIN_REGEX.test(value); +// }, +// { +// message: "Invalid domain name or IP address format", +// path: ["domain"] +// } +// ); const createTargetParamsSchema = z .object({ @@ -52,7 +52,7 @@ const createTargetParamsSchema = z const createTargetSchema = z .object({ - ip: domainSchema, + ip: z.string().min(1).max(255), method: z.string().optional().nullable(), port: z.number().int().min(1).max(65535), enabled: z.boolean().default(true) diff --git a/server/routers/target/updateTarget.ts b/server/routers/target/updateTarget.ts index 2ae6222d..4dbb2f45 100644 --- a/server/routers/target/updateTarget.ts +++ b/server/routers/target/updateTarget.ts @@ -12,33 +12,33 @@ import { addPeer } from "../gerbil/peers"; import { addTargets } from "../newt/targets"; import { pickPort } from "./helpers"; -// Regular expressions for validation -const DOMAIN_REGEX = - /^[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$/; -const IPV4_REGEX = - /^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$/; -const IPV6_REGEX = /^(?:[A-F0-9]{1,4}:){7}[A-F0-9]{1,4}$/i; - -// Schema for domain names and IP addresses -const domainSchema = z - .string() - .min(1, "Domain cannot be empty") - .max(255, "Domain name too long") - .refine( - (value) => { - // Check if it's a valid IP address (v4 or v6) - if (IPV4_REGEX.test(value) || IPV6_REGEX.test(value)) { - return true; - } - - // Check if it's a valid domain name - return DOMAIN_REGEX.test(value); - }, - { - message: "Invalid domain name or IP address format", - path: ["domain"] - } - ); +// // Regular expressions for validation +// const DOMAIN_REGEX = +// /^[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$/; +// const IPV4_REGEX = +// /^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$/; +// const IPV6_REGEX = /^(?:[A-F0-9]{1,4}:){7}[A-F0-9]{1,4}$/i; +// +// // Schema for domain names and IP addresses +// const domainSchema = z +// .string() +// .min(1, "Domain cannot be empty") +// .max(255, "Domain name too long") +// .refine( +// (value) => { +// // Check if it's a valid IP address (v4 or v6) +// if (IPV4_REGEX.test(value) || IPV6_REGEX.test(value)) { +// return true; +// } +// +// // Check if it's a valid domain name +// return DOMAIN_REGEX.test(value); +// }, +// { +// message: "Invalid domain name or IP address format", +// path: ["domain"] +// } +// ); const updateTargetParamsSchema = z .object({ @@ -48,7 +48,7 @@ const updateTargetParamsSchema = z const updateTargetBodySchema = z .object({ - ip: domainSchema.optional(), + ip: z.string().min(1).max(255), method: z.string().min(1).max(10).optional().nullable(), port: z.number().int().min(1).max(65535).optional(), enabled: z.boolean().optional() diff --git a/src/app/[orgId]/settings/resources/[resourceId]/connectivity/page.tsx b/src/app/[orgId]/settings/resources/[resourceId]/connectivity/page.tsx index ea67e23e..d912b505 100644 --- a/src/app/[orgId]/settings/resources/[resourceId]/connectivity/page.tsx +++ b/src/app/[orgId]/settings/resources/[resourceId]/connectivity/page.tsx @@ -73,29 +73,29 @@ const IPV4_REGEX = /^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$/; const IPV6_REGEX = /^(?:[A-F0-9]{1,4}:){7}[A-F0-9]{1,4}$/i; -// Schema for domain names and IP addresses -const domainSchema = z - .string() - .min(1, "Domain cannot be empty") - .max(255, "Domain name too long") - .refine( - (value) => { - // Check if it's a valid IP address (v4 or v6) - if (IPV4_REGEX.test(value) || IPV6_REGEX.test(value)) { - return true; - } - - // Check if it's a valid domain name - return DOMAIN_REGEX.test(value); - }, - { - message: "Invalid domain name or IP address format", - path: ["domain"] - } - ); +// // Schema for domain names and IP addresses +// const domainSchema = z +// .string() +// .min(1, "Domain cannot be empty") +// .max(255, "Domain name too long") +// .refine( +// (value) => { +// // Check if it's a valid IP address (v4 or v6) +// if (IPV4_REGEX.test(value) || IPV6_REGEX.test(value)) { +// return true; +// } +// +// // Check if it's a valid domain name +// return DOMAIN_REGEX.test(value); +// }, +// { +// message: "Invalid domain name or IP address format", +// path: ["domain"] +// } +// ); const addTargetSchema = z.object({ - ip: domainSchema, + ip: z.string().min(1).max(255), method: z.string().nullable(), port: z.coerce.number().int().positive() // protocol: z.string(), From a418195b283d0f9825bba34607e714cbab999b2e Mon Sep 17 00:00:00 2001 From: Owen Date: Fri, 14 Feb 2025 15:49:23 -0500 Subject: [PATCH 11/26] Fix ip range pick initial range; add test --- server/lib/ip.test.ts | 183 ++++++++++++++++++++++++++++++++++++++++++ server/lib/ip.ts | 19 +++-- 2 files changed, 196 insertions(+), 6 deletions(-) create mode 100644 server/lib/ip.test.ts diff --git a/server/lib/ip.test.ts b/server/lib/ip.test.ts new file mode 100644 index 00000000..f3925cf1 --- /dev/null +++ b/server/lib/ip.test.ts @@ -0,0 +1,183 @@ +import { cidrToRange, findNextAvailableCidr } from "./ip"; + +/** + * Compares two objects for deep equality + * @param actual The actual value to test + * @param expected The expected value to compare against + * @param message The message to display if assertion fails + * @throws Error if objects are not equal + */ +export function assertEqualsObj(actual: T, expected: T, message: string): void { + const actualStr = JSON.stringify(actual); + const expectedStr = JSON.stringify(expected); + if (actualStr !== expectedStr) { + throw new Error(`${message}\nExpected: ${expectedStr}\nActual: ${actualStr}`); + } +} + +/** + * Compares two primitive values for equality + * @param actual The actual value to test + * @param expected The expected value to compare against + * @param message The message to display if assertion fails + * @throws Error if values are not equal + */ +export function assertEquals(actual: T, expected: T, message: string): void { + if (actual !== expected) { + throw new Error(`${message}\nExpected: ${expected}\nActual: ${actual}`); + } +} + +/** + * Tests if a function throws an expected error + * @param fn The function to test + * @param expectedError The expected error message or part of it + * @param message The message to display if assertion fails + * @throws Error if function doesn't throw or throws unexpected error + */ +export function assertThrows( + fn: () => void, + expectedError: string, + message: string +): void { + try { + fn(); + throw new Error(`${message}: Expected to throw "${expectedError}"`); + } catch (error) { + if (!(error instanceof Error)) { + throw new Error(`${message}\nUnexpected error type: ${typeof error}`); + } + + if (!error.message.includes(expectedError)) { + throw new Error( + `${message}\nExpected error: ${expectedError}\nActual error: ${error.message}` + ); + } + } +} + + +// Test cases +function testFindNextAvailableCidr() { + console.log("Running findNextAvailableCidr tests..."); + + // Test 1: Basic IPv4 allocation + { + const existing = ["10.0.0.0/16", "10.1.0.0/16"]; + const result = findNextAvailableCidr(existing, 16, "10.0.0.0/8"); + assertEquals(result, "10.2.0.0/16", "Basic IPv4 allocation failed"); + } + + // Test 2: Finding gap between allocations + { + const existing = ["10.0.0.0/16", "10.2.0.0/16"]; + const result = findNextAvailableCidr(existing, 16, "10.0.0.0/8"); + assertEquals(result, "10.1.0.0/16", "Finding gap between allocations failed"); + } + + // Test 3: No available space + { + const existing = ["10.0.0.0/8"]; + const result = findNextAvailableCidr(existing, 8, "10.0.0.0/8"); + assertEquals(result, null, "No available space test failed"); + } + + // // Test 4: IPv6 allocation + // { + // const existing = ["2001:db8::/32", "2001:db8:1::/32"]; + // const result = findNextAvailableCidr(existing, 32, "2001:db8::/16"); + // assertEquals(result, "2001:db8:2::/32", "Basic IPv6 allocation failed"); + // } + + // // Test 5: Mixed IP versions + // { + // const existing = ["10.0.0.0/16", "2001:db8::/32"]; + // assertThrows( + // () => findNextAvailableCidr(existing, 16), + // "All CIDRs must be of the same IP version", + // "Mixed IP versions test failed" + // ); + // } + + // Test 6: Empty input + { + const existing: string[] = []; + const result = findNextAvailableCidr(existing, 16); + assertEquals(result, null, "Empty input test failed"); + } + + // Test 7: Block size alignment + { + const existing = ["10.0.0.0/24"]; + const result = findNextAvailableCidr(existing, 24, "10.0.0.0/16"); + assertEquals(result, "10.0.1.0/24", "Block size alignment test failed"); + } + + // Test 8: Block size alignment + { + const existing: string[] = []; + const result = findNextAvailableCidr(existing, 24, "10.0.0.0/16"); + assertEquals(result, "10.0.0.0/24", "Block size alignment test failed"); + } + + // Test 9: Large block size request + { + const existing = ["10.0.0.0/24", "10.0.1.0/24"]; + const result = findNextAvailableCidr(existing, 16, "10.0.0.0/16"); + assertEquals(result, null, "Large block size request test failed"); + } + + console.log("All findNextAvailableCidr tests passed!"); +} + +// function testCidrToRange() { +// console.log("Running cidrToRange tests..."); + +// // Test 1: Basic IPv4 conversion +// { +// const result = cidrToRange("192.168.0.0/24"); +// assertEqualsObj(result, { +// start: BigInt("3232235520"), +// end: BigInt("3232235775") +// }, "Basic IPv4 conversion failed"); +// } + +// // Test 2: IPv6 conversion +// { +// const result = cidrToRange("2001:db8::/32"); +// assertEqualsObj(result, { +// start: BigInt("42540766411282592856903984951653826560"), +// end: BigInt("42540766411282592875350729025363378175") +// }, "IPv6 conversion failed"); +// } + +// // Test 3: Invalid prefix length +// { +// assertThrows( +// () => cidrToRange("192.168.0.0/33"), +// "Invalid prefix length for IPv4", +// "Invalid IPv4 prefix test failed" +// ); +// } + +// // Test 4: Invalid IPv6 prefix +// { +// assertThrows( +// () => cidrToRange("2001:db8::/129"), +// "Invalid prefix length for IPv6", +// "Invalid IPv6 prefix test failed" +// ); +// } + +// console.log("All cidrToRange tests passed!"); +// } + +// Run all tests +try { + // testCidrToRange(); + testFindNextAvailableCidr(); + console.log("All tests passed successfully!"); +} catch (error) { + console.error("Test failed:", error); + process.exit(1); +} \ No newline at end of file diff --git a/server/lib/ip.ts b/server/lib/ip.ts index 62cf1d2d..86fe1169 100644 --- a/server/lib/ip.ts +++ b/server/lib/ip.ts @@ -100,7 +100,7 @@ function bigIntToIp(num: bigint, version: IPVersion): string { /** * Converts CIDR to IP range */ -function cidrToRange(cidr: string): IPRange { +export function cidrToRange(cidr: string): IPRange { const [ip, prefix] = cidr.split('/'); const version = detectIpVersion(ip); const prefixBits = parseInt(prefix); @@ -132,15 +132,22 @@ export function findNextAvailableCidr( blockSize: number, startCidr?: string ): string | null { - if (existingCidrs.length === 0) return null; + + if (!startCidr && existingCidrs.length === 0) { + return null; + } + + // If no existing CIDRs, use the IP version from startCidr + const version = startCidr + ? detectIpVersion(startCidr.split('/')[0]) + : 4; // Default to IPv4 if no startCidr provided - // Determine IP version from first CIDR - const version = detectIpVersion(existingCidrs[0].split('/')[0]); // Use appropriate default startCidr if none provided startCidr = startCidr || (version === 4 ? "0.0.0.0/0" : "::/0"); - // Ensure all CIDRs are same version - if (existingCidrs.some(cidr => detectIpVersion(cidr.split('/')[0]) !== version)) { + // If there are existing CIDRs, ensure all are same version + if (existingCidrs.length > 0 && + existingCidrs.some(cidr => detectIpVersion(cidr.split('/')[0]) !== version)) { throw new Error('All CIDRs must be of the same IP version'); } From d5a220a0047cf76afe6b7639f7510b7a1ecc0684 Mon Sep 17 00:00:00 2001 From: Milo Schwartz Date: Fri, 14 Feb 2025 16:46:46 -0500 Subject: [PATCH 12/26] create target validator and add url validator --- server/lib/validators.ts | 47 ++++++++++++++++--- server/routers/target/createTarget.ts | 31 +----------- server/routers/target/updateTarget.ts | 31 +----------- .../[resourceId]/connectivity/page.tsx | 33 +------------ 4 files changed, 46 insertions(+), 96 deletions(-) diff --git a/server/lib/validators.ts b/server/lib/validators.ts index 675c0809..abb2ebb4 100644 --- a/server/lib/validators.ts +++ b/server/lib/validators.ts @@ -11,7 +11,7 @@ export function isValidIP(ip: string): boolean { export function isValidUrlGlobPattern(pattern: string): boolean { // Remove leading slash if present pattern = pattern.startsWith("/") ? pattern.slice(1) : pattern; - + // Empty string is not valid if (!pattern) { return false; @@ -19,11 +19,11 @@ export function isValidUrlGlobPattern(pattern: string): boolean { // Split path into segments const segments = pattern.split("/"); - + // Check each segment for (let i = 0; i < segments.length; i++) { const segment = segments[i]; - + // Empty segments are not allowed (double slashes), except at the end if (!segment && i !== segments.length - 1) { return false; @@ -37,12 +37,15 @@ export function isValidUrlGlobPattern(pattern: string): boolean { // Check each character in the segment for (let j = 0; j < segment.length; j++) { const char = segment[j]; - + // Check for percent-encoded sequences if (char === "%" && j + 2 < segment.length) { const hex1 = segment[j + 1]; const hex2 = segment[j + 2]; - if (!/^[0-9A-Fa-f]$/.test(hex1) || !/^[0-9A-Fa-f]$/.test(hex2)) { + if ( + !/^[0-9A-Fa-f]$/.test(hex1) || + !/^[0-9A-Fa-f]$/.test(hex2) + ) { return false; } j += 2; // Skip the next two characters @@ -58,6 +61,36 @@ export function isValidUrlGlobPattern(pattern: string): boolean { } } } - + return true; -} \ No newline at end of file +} + +export function isUrlValid(url: string | undefined) { + if (!url) return true; // the link is optional in the schema so if it's empty it's valid + var pattern = new RegExp( + "^(https?:\\/\\/)?" + // protocol + "((([a-z\\d]([a-z\\d-]*[a-z\\d])*)\\.)+[a-z]{2,}|" + // domain name + "((\\d{1,3}\\.){3}\\d{1,3}))" + // OR ip (v4) address + "(\\:\\d+)?(\\/[-a-z\\d%_.~+]*)*" + // port and path + "(\\?[;&a-z\\d%_.~+=-]*)?" + // query string + "(\\#[-a-z\\d_]*)?$", + "i" + ); + return !!pattern.test(url); +} + +export function isTargetValid(value: string | undefined) { + if (!value) return true; + + const DOMAIN_REGEX = + /^[a-zA-Z0-9_](?:[a-zA-Z0-9-_]{0,61}[a-zA-Z0-9_])?(?:\.[a-zA-Z0-9_](?:[a-zA-Z0-9-_]{0,61}[a-zA-Z0-9_])?)*$/; + const IPV4_REGEX = + /^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$/; + const IPV6_REGEX = /^(?:[A-F0-9]{1,4}:){7}[A-F0-9]{1,4}$/i; + + if (IPV4_REGEX.test(value) || IPV6_REGEX.test(value)) { + return true; + } + + return DOMAIN_REGEX.test(value); +} diff --git a/server/routers/target/createTarget.ts b/server/routers/target/createTarget.ts index 11f3de69..8d07e5d6 100644 --- a/server/routers/target/createTarget.ts +++ b/server/routers/target/createTarget.ts @@ -12,34 +12,7 @@ import { fromError } from "zod-validation-error"; import { addTargets } from "../newt/targets"; import { eq } from "drizzle-orm"; import { pickPort } from "./helpers"; - -// // Regular expressions for validation -// const DOMAIN_REGEX = -// /^[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$/; -// const IPV4_REGEX = -// /^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$/; -// const IPV6_REGEX = /^(?:[A-F0-9]{1,4}:){7}[A-F0-9]{1,4}$/i; -// -// // Schema for domain names and IP addresses -// const domainSchema = z -// .string() -// .min(1, "Domain cannot be empty") -// .max(255, "Domain name too long") -// .refine( -// (value) => { -// // Check if it's a valid IP address (v4 or v6) -// if (IPV4_REGEX.test(value) || IPV6_REGEX.test(value)) { -// return true; -// } -// -// // Check if it's a valid domain name -// return DOMAIN_REGEX.test(value); -// }, -// { -// message: "Invalid domain name or IP address format", -// path: ["domain"] -// } -// ); +import { isTargetValid } from "@server/lib/validators"; const createTargetParamsSchema = z .object({ @@ -52,7 +25,7 @@ const createTargetParamsSchema = z const createTargetSchema = z .object({ - ip: z.string().min(1).max(255), + ip: z.string().refine(isTargetValid), method: z.string().optional().nullable(), port: z.number().int().min(1).max(65535), enabled: z.boolean().default(true) diff --git a/server/routers/target/updateTarget.ts b/server/routers/target/updateTarget.ts index 4dbb2f45..45051e0a 100644 --- a/server/routers/target/updateTarget.ts +++ b/server/routers/target/updateTarget.ts @@ -11,34 +11,7 @@ import { fromError } from "zod-validation-error"; import { addPeer } from "../gerbil/peers"; import { addTargets } from "../newt/targets"; import { pickPort } from "./helpers"; - -// // Regular expressions for validation -// const DOMAIN_REGEX = -// /^[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$/; -// const IPV4_REGEX = -// /^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$/; -// const IPV6_REGEX = /^(?:[A-F0-9]{1,4}:){7}[A-F0-9]{1,4}$/i; -// -// // Schema for domain names and IP addresses -// const domainSchema = z -// .string() -// .min(1, "Domain cannot be empty") -// .max(255, "Domain name too long") -// .refine( -// (value) => { -// // Check if it's a valid IP address (v4 or v6) -// if (IPV4_REGEX.test(value) || IPV6_REGEX.test(value)) { -// return true; -// } -// -// // Check if it's a valid domain name -// return DOMAIN_REGEX.test(value); -// }, -// { -// message: "Invalid domain name or IP address format", -// path: ["domain"] -// } -// ); +import { isTargetValid } from "@server/lib/validators"; const updateTargetParamsSchema = z .object({ @@ -48,7 +21,7 @@ const updateTargetParamsSchema = z const updateTargetBodySchema = z .object({ - ip: z.string().min(1).max(255), + ip: z.string().refine(isTargetValid), method: z.string().min(1).max(10).optional().nullable(), port: z.number().int().min(1).max(65535).optional(), enabled: z.boolean().optional() diff --git a/src/app/[orgId]/settings/resources/[resourceId]/connectivity/page.tsx b/src/app/[orgId]/settings/resources/[resourceId]/connectivity/page.tsx index d912b505..c565b525 100644 --- a/src/app/[orgId]/settings/resources/[resourceId]/connectivity/page.tsx +++ b/src/app/[orgId]/settings/resources/[resourceId]/connectivity/page.tsx @@ -62,40 +62,11 @@ import { SettingsSectionFooter } from "@app/components/Settings"; import { SwitchInput } from "@app/components/SwitchInput"; -import { useSiteContext } from "@app/hooks/useSiteContext"; -import { InfoPopup } from "@app/components/ui/info-popup"; import { useRouter } from "next/navigation"; - -// Regular expressions for validation -const DOMAIN_REGEX = - /^[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$/; -const IPV4_REGEX = - /^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$/; -const IPV6_REGEX = /^(?:[A-F0-9]{1,4}:){7}[A-F0-9]{1,4}$/i; - -// // Schema for domain names and IP addresses -// const domainSchema = z -// .string() -// .min(1, "Domain cannot be empty") -// .max(255, "Domain name too long") -// .refine( -// (value) => { -// // Check if it's a valid IP address (v4 or v6) -// if (IPV4_REGEX.test(value) || IPV6_REGEX.test(value)) { -// return true; -// } -// -// // Check if it's a valid domain name -// return DOMAIN_REGEX.test(value); -// }, -// { -// message: "Invalid domain name or IP address format", -// path: ["domain"] -// } -// ); +import { isTargetValid } from "@server/lib/validators"; const addTargetSchema = z.object({ - ip: z.string().min(1).max(255), + ip: z.string().refine(isTargetValid), method: z.string().nullable(), port: z.coerce.number().int().positive() // protocol: z.string(), From 6aa49084465df74f2da5c6d59b1168bb3cb210ac Mon Sep 17 00:00:00 2001 From: Milo Schwartz Date: Fri, 14 Feb 2025 16:53:05 -0500 Subject: [PATCH 13/26] bump version --- server/lib/consts.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/lib/consts.ts b/server/lib/consts.ts index 20376f8e..e502ccc8 100644 --- a/server/lib/consts.ts +++ b/server/lib/consts.ts @@ -2,7 +2,7 @@ import path from "path"; import { fileURLToPath } from "url"; // This is a placeholder value replaced by the build process -export const APP_VERSION = "1.0.0-beta.13"; +export const APP_VERSION = "1.0.0-beta.14"; export const __FILENAME = fileURLToPath(import.meta.url); export const __DIRNAME = path.dirname(__FILENAME); From bdee036ab422683c9534d9a2d589d93702972bd2 Mon Sep 17 00:00:00 2001 From: Owen Date: Sat, 15 Feb 2025 17:08:11 -0500 Subject: [PATCH 14/26] Add name; Resolves #190 --- docker-compose.example.yml | 8 +++----- install/fs/docker-compose.yml | 1 + 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/docker-compose.example.yml b/docker-compose.example.yml index bc5ad10c..ad755174 100644 --- a/docker-compose.example.yml +++ b/docker-compose.example.yml @@ -1,5 +1,4 @@ -version: "3.7" - +name: pangolin services: pangolin: image: fosrl/pangolin:latest @@ -32,7 +31,6 @@ services: - SYS_MODULE ports: - 51820:51820/udp - - 8080:8080 # Port for traefik because of the network_mode - 443:443 # Port for traefik because of the network_mode - 80:80 # Port for traefik because of the network_mode @@ -47,8 +45,8 @@ services: command: - --configFile=/etc/traefik/traefik_config.yml volumes: - - ./traefik:/etc/traefik:ro # Volume to store the Traefik configuration - - ./letsencrypt:/letsencrypt # Volume to store the Let's Encrypt certificates + - ./config/traefik:/etc/traefik:ro # Volume to store the Traefik configuration + - ./config/letsencrypt:/letsencrypt # Volume to store the Let's Encrypt certificates networks: default: diff --git a/install/fs/docker-compose.yml b/install/fs/docker-compose.yml index ea673eb0..b26e0257 100644 --- a/install/fs/docker-compose.yml +++ b/install/fs/docker-compose.yml @@ -1,3 +1,4 @@ +name: pangolin services: pangolin: image: fosrl/pangolin:{{.PangolinVersion}} From b862e1aeef667ccd7e0157639d95dcea6e14d48f Mon Sep 17 00:00:00 2001 From: Owen Date: Sat, 15 Feb 2025 17:43:44 -0500 Subject: [PATCH 15/26] Add h2c as target method; Resolves #115 --- .../settings/resources/[resourceId]/connectivity/page.tsx | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/app/[orgId]/settings/resources/[resourceId]/connectivity/page.tsx b/src/app/[orgId]/settings/resources/[resourceId]/connectivity/page.tsx index c565b525..67434404 100644 --- a/src/app/[orgId]/settings/resources/[resourceId]/connectivity/page.tsx +++ b/src/app/[orgId]/settings/resources/[resourceId]/connectivity/page.tsx @@ -421,6 +421,7 @@ export default function ReverseProxyTargets(props: { http https + h2c ) @@ -517,6 +518,9 @@ export default function ReverseProxyTargets(props: { https + + h2c + From 7bf820a4bfd19c5278e2d39463151fe1aa5fd7f8 Mon Sep 17 00:00:00 2001 From: Owen Date: Sat, 15 Feb 2025 17:48:27 -0500 Subject: [PATCH 16/26] Clean off ports for 80 and 443 hosts --- server/routers/badger/verifySession.ts | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/server/routers/badger/verifySession.ts b/server/routers/badger/verifySession.ts index fc1c85f5..69314cbc 100644 --- a/server/routers/badger/verifySession.ts +++ b/server/routers/badger/verifySession.ts @@ -90,7 +90,15 @@ export async function verifyResourceSession( const clientIp = requestIp?.split(":")[0]; - const resourceCacheKey = `resource:${host}`; + let cleanHost = host; + // if the host ends with :443 or :80 remove it + if (cleanHost.endsWith(":443")) { + cleanHost = cleanHost.slice(0, -4); + } else if (cleanHost.endsWith(":80")) { + cleanHost = cleanHost.slice(0, -3); + } + + const resourceCacheKey = `resource:${cleanHost}`; let resourceData: | { resource: Resource | null; @@ -111,11 +119,11 @@ export async function verifyResourceSession( resourcePassword, eq(resourcePassword.resourceId, resources.resourceId) ) - .where(eq(resources.fullDomain, host)) + .where(eq(resources.fullDomain, cleanHost)) .limit(1); if (!result) { - logger.debug("Resource not found", host); + logger.debug("Resource not found", cleanHost); return notAllowed(res); } @@ -131,7 +139,7 @@ export async function verifyResourceSession( const { resource, pincode, password } = resourceData; if (!resource) { - logger.debug("Resource not found", host); + logger.debug("Resource not found", cleanHost); return notAllowed(res); } From dabd4a055c2c63996c585018ce3e7d696a3d7868 Mon Sep 17 00:00:00 2001 From: Owen Date: Sat, 15 Feb 2025 22:00:31 -0500 Subject: [PATCH 17/26] Creating structure correctly --- install/config.go | 2 +- install/crowdsec.go | 4 ++-- install/input.txt | 12 ++++++++++++ install/main.go | 38 ++++++++++++++++++++------------------ 4 files changed, 35 insertions(+), 21 deletions(-) create mode 100644 install/input.txt diff --git a/install/config.go b/install/config.go index fa10fa53..49577048 100644 --- a/install/config.go +++ b/install/config.go @@ -164,7 +164,7 @@ func AddCrowdSecService(configPath string) error { Image: "crowdsecurity/crowdsec:latest", ContainerName: "crowdsec", Environment: map[string]string{ - "GID": "${GID-1000}", + "GID": "1000", "COLLECTIONS": "crowdsecurity/traefik crowdsecurity/appsec-virtual-patching crowdsecurity/appsec-generic-rules", "ENROLL_INSTANCE_NAME": "pangolin-crowdsec", "PARSERS": "crowdsecurity/whitelists", diff --git a/install/crowdsec.go b/install/crowdsec.go index 02e9c71b..1d9bef65 100644 --- a/install/crowdsec.go +++ b/install/crowdsec.go @@ -24,8 +24,8 @@ func installCrowdsec(config Config) error { os.Exit(1) } - moveFile("config/crowdsec/traefik_config.yaml", "config/traefik/traefik_config.yaml") - moveFile("config/crowdsec/dynamic.yaml", "config/traefik/dynamic.yaml") + // moveFile("config/crowdsec/traefik_config.yml", "config/traefik/traefik_config.yml") + moveFile("config/crowdsec/dynamic.yml", "config/traefik/dynamic.yml") if err := retrieveBouncerKey(config); err != nil { return fmt.Errorf("bouncer key retrieval failed: %v", err) diff --git a/install/input.txt b/install/input.txt new file mode 100644 index 00000000..9bca8081 --- /dev/null +++ b/install/input.txt @@ -0,0 +1,12 @@ +example.com +pangolin.example.com +admin@example.com +yes +admin@example.com +Password123! +Password123! +yes +no +no +no +yes diff --git a/install/main.go b/install/main.go index 4f7c7df6..e5ad98b9 100644 --- a/install/main.go +++ b/install/main.go @@ -140,22 +140,24 @@ func readString(reader *bufio.Reader, prompt string, defaultValue string) string return input } -func readPassword(prompt string) string { - fmt.Print(prompt + ": ") - - // Read password without echo - password, err := term.ReadPassword(int(syscall.Stdin)) - fmt.Println() // Add a newline since ReadPassword doesn't add one - - if err != nil { - return "" - } - - input := strings.TrimSpace(string(password)) - if input == "" { - return readPassword(prompt) - } - return input +func readPassword(prompt string, reader *bufio.Reader) string { + if term.IsTerminal(int(syscall.Stdin)) { + fmt.Print(prompt + ": ") + // Read password without echo if we're in a terminal + password, err := term.ReadPassword(int(syscall.Stdin)) + fmt.Println() // Add a newline since ReadPassword doesn't add one + if err != nil { + return "" + } + input := strings.TrimSpace(string(password)) + if input == "" { + return readPassword(prompt, reader) + } + return input + } else { + // Fallback to reading from stdin if not in a terminal + return readString(reader, prompt, "") + } } func readBool(reader *bufio.Reader, prompt string, defaultValue bool) bool { @@ -196,8 +198,8 @@ func collectUserInput(reader *bufio.Reader) Config { fmt.Println("\n=== Admin User Configuration ===") config.AdminUserEmail = readString(reader, "Enter admin user email", "admin@"+config.BaseDomain) for { - pass1 := readPassword("Create admin user password") - pass2 := readPassword("Confirm admin user password") + pass1 := readPassword("Create admin user password", reader) + pass2 := readPassword("Confirm admin user password", reader) if pass1 != pass2 { fmt.Println("Passwords do not match") From d3d523b2b8f1de0d25d176be134515a0614b0721 Mon Sep 17 00:00:00 2001 From: Owen Date: Sun, 16 Feb 2025 11:26:45 -0500 Subject: [PATCH 18/26] Refactor docker copy and keep entrypoints --- install/config.go | 268 +++++++++++++-------- install/config/crowdsec/docker-compose.yml | 35 +++ install/config/crowdsec/traefik_config.yml | 2 +- install/crowdsec.go | 35 ++- 4 files changed, 230 insertions(+), 110 deletions(-) create mode 100644 install/config/crowdsec/docker-compose.yml diff --git a/install/config.go b/install/config.go index 49577048..1ebfbecb 100644 --- a/install/config.go +++ b/install/config.go @@ -104,113 +104,175 @@ func findPattern(s, pattern string) int { return bytes.Index([]byte(s), []byte(pattern)) } -type Volume string -type Port string -type Expose string - -type HealthCheck struct { - Test []string `yaml:"test,omitempty"` - Interval string `yaml:"interval,omitempty"` - Timeout string `yaml:"timeout,omitempty"` - Retries int `yaml:"retries,omitempty"` -} - -type DependsOnCondition struct { - Condition string `yaml:"condition,omitempty"` -} - -type Service struct { - Image string `yaml:"image,omitempty"` - ContainerName string `yaml:"container_name,omitempty"` - Environment map[string]string `yaml:"environment,omitempty"` - HealthCheck *HealthCheck `yaml:"healthcheck,omitempty"` - DependsOn map[string]DependsOnCondition `yaml:"depends_on,omitempty"` - Labels []string `yaml:"labels,omitempty"` - Volumes []Volume `yaml:"volumes,omitempty"` - Ports []Port `yaml:"ports,omitempty"` - Expose []Expose `yaml:"expose,omitempty"` - Restart string `yaml:"restart,omitempty"` - Command interface{} `yaml:"command,omitempty"` - NetworkMode string `yaml:"network_mode,omitempty"` - CapAdd []string `yaml:"cap_add,omitempty"` -} - -type Network struct { - Driver string `yaml:"driver,omitempty"` - Name string `yaml:"name,omitempty"` -} - -type DockerConfig struct { - Version string `yaml:"version,omitempty"` - Services map[string]Service `yaml:"services"` - Networks map[string]Network `yaml:"networks,omitempty"` -} - -func AddCrowdSecService(configPath string) error { - // Read existing config - data, err := os.ReadFile(configPath) +func copyEntryPoints(sourceFile, destFile string) error { + // Read source file + sourceData, err := os.ReadFile(sourceFile) if err != nil { - return err + return fmt.Errorf("error reading source file: %w", err) } - // Parse existing config - var config DockerConfig - if err := yaml.Unmarshal(data, &config); err != nil { - return err - } - - // Create CrowdSec service - crowdsecService := Service{ - Image: "crowdsecurity/crowdsec:latest", - ContainerName: "crowdsec", - Environment: map[string]string{ - "GID": "1000", - "COLLECTIONS": "crowdsecurity/traefik crowdsecurity/appsec-virtual-patching crowdsecurity/appsec-generic-rules", - "ENROLL_INSTANCE_NAME": "pangolin-crowdsec", - "PARSERS": "crowdsecurity/whitelists", - "ACQUIRE_FILES": "/var/log/traefik/*.log", - "ENROLL_TAGS": "docker", - }, - HealthCheck: &HealthCheck{ - Test: []string{"CMD", "cscli", "capi", "status"}, - }, - DependsOn: map[string]DependsOnCondition{ - "gerbil": {}, - }, - Labels: []string{"traefik.enable=false"}, - Volumes: []Volume{ - "./config/crowdsec:/etc/crowdsec", - "./config/crowdsec/db:/var/lib/crowdsec/data", - "./config/crowdsec_logs/auth.log:/var/log/auth.log:ro", - "./config/crowdsec_logs/syslog:/var/log/syslog:ro", - "./config/crowdsec_logs:/var/log", - "./config/traefik/logs:/var/log/traefik", - }, - Ports: []Port{ - "9090:9090", - "6060:6060", - }, - Expose: []Expose{ - "9090", - "6060", - "7422", - }, - Restart: "unless-stopped", - Command: "-t", - } - - // Add CrowdSec service to config - if config.Services == nil { - config.Services = make(map[string]Service) - } - config.Services["crowdsec"] = crowdsecService - - // Marshal config with better formatting - yamlData, err := yaml.Marshal(&config) + // Read destination file + destData, err := os.ReadFile(destFile) if err != nil { - return err + return fmt.Errorf("error reading destination file: %w", err) } - // Write config back to file - return os.WriteFile(configPath, yamlData, 0644) + // Parse source YAML + var sourceYAML map[string]interface{} + if err := yaml.Unmarshal(sourceData, &sourceYAML); err != nil { + return fmt.Errorf("error parsing source YAML: %w", err) + } + + // Parse destination YAML + var destYAML map[string]interface{} + if err := yaml.Unmarshal(destData, &destYAML); err != nil { + return fmt.Errorf("error parsing destination YAML: %w", err) + } + + // Get entryPoints section from source + entryPoints, ok := sourceYAML["entryPoints"] + if !ok { + return fmt.Errorf("entryPoints section not found in source file") + } + + // Update entryPoints in destination + destYAML["entryPoints"] = entryPoints + + // Marshal updated destination YAML + updatedData, err := yaml.Marshal(destYAML) + if err != nil { + return fmt.Errorf("error marshaling updated YAML: %w", err) + } + + // Write updated YAML back to destination file + if err := os.WriteFile(destFile, updatedData, 0644); err != nil { + return fmt.Errorf("error writing to destination file: %w", err) + } + + return nil +} + +func copyWebsecureEntryPoint(sourceFile, destFile string) error { + // Read source file + sourceData, err := os.ReadFile(sourceFile) + if err != nil { + return fmt.Errorf("error reading source file: %w", err) + } + + // Read destination file + destData, err := os.ReadFile(destFile) + if err != nil { + return fmt.Errorf("error reading destination file: %w", err) + } + + // Parse source YAML + var sourceYAML map[string]interface{} + if err := yaml.Unmarshal(sourceData, &sourceYAML); err != nil { + return fmt.Errorf("error parsing source YAML: %w", err) + } + + // Parse destination YAML + var destYAML map[string]interface{} + if err := yaml.Unmarshal(destData, &destYAML); err != nil { + return fmt.Errorf("error parsing destination YAML: %w", err) + } + + // Get entryPoints section from source + entryPoints, ok := sourceYAML["entryPoints"].(map[string]interface{}) + if !ok { + return fmt.Errorf("entryPoints section not found in source file or has invalid format") + } + + // Get websecure configuration + websecure, ok := entryPoints["websecure"] + if !ok { + return fmt.Errorf("websecure entrypoint not found in source file") + } + + // Get or create entryPoints section in destination + destEntryPoints, ok := destYAML["entryPoints"].(map[string]interface{}) + if !ok { + // If entryPoints section doesn't exist, create it + destEntryPoints = make(map[string]interface{}) + destYAML["entryPoints"] = destEntryPoints + } + + // Update websecure in destination + destEntryPoints["websecure"] = websecure + + // Marshal updated destination YAML + updatedData, err := yaml.Marshal(destYAML) + if err != nil { + return fmt.Errorf("error marshaling updated YAML: %w", err) + } + + // Write updated YAML back to destination file + if err := os.WriteFile(destFile, updatedData, 0644); err != nil { + return fmt.Errorf("error writing to destination file: %w", err) + } + + return nil +} + +func copyDockerService(sourceFile, destFile, serviceName string) error { + // Read source file + sourceData, err := os.ReadFile(sourceFile) + if err != nil { + return fmt.Errorf("error reading source file: %w", err) + } + + // Read destination file + destData, err := os.ReadFile(destFile) + if err != nil { + return fmt.Errorf("error reading destination file: %w", err) + } + + // Parse source Docker Compose YAML + var sourceCompose map[string]interface{} + if err := yaml.Unmarshal(sourceData, &sourceCompose); err != nil { + return fmt.Errorf("error parsing source Docker Compose file: %w", err) + } + + // Parse destination Docker Compose YAML + var destCompose map[string]interface{} + if err := yaml.Unmarshal(destData, &destCompose); err != nil { + return fmt.Errorf("error parsing destination Docker Compose file: %w", err) + } + + // Get services section from source + sourceServices, ok := sourceCompose["services"].(map[string]interface{}) + if !ok { + return fmt.Errorf("services section not found in source file or has invalid format") + } + + // Get the specific service configuration + serviceConfig, ok := sourceServices[serviceName] + if !ok { + return fmt.Errorf("service '%s' not found in source file", serviceName) + } + + // Get or create services section in destination + destServices, ok := destCompose["services"].(map[string]interface{}) + if !ok { + // If services section doesn't exist, create it + destServices = make(map[string]interface{}) + destCompose["services"] = destServices + } + + // Update service in destination + destServices[serviceName] = serviceConfig + + // Marshal updated destination YAML + // Use yaml.v3 encoder to preserve formatting and comments + updatedData, err := yaml.Marshal(destCompose) + if err != nil { + return fmt.Errorf("error marshaling updated Docker Compose file: %w", err) + } + + // Write updated YAML back to destination file + if err := os.WriteFile(destFile, updatedData, 0644); err != nil { + return fmt.Errorf("error writing to destination file: %w", err) + } + + return nil } diff --git a/install/config/crowdsec/docker-compose.yml b/install/config/crowdsec/docker-compose.yml new file mode 100644 index 00000000..982b3335 --- /dev/null +++ b/install/config/crowdsec/docker-compose.yml @@ -0,0 +1,35 @@ +services: + crowdsec: + image: crowdsecurity/crowdsec:latest + container_name: crowdsec + environment: + GID: "1000" + COLLECTIONS: crowdsecurity/traefik crowdsecurity/appsec-virtual-patching crowdsecurity/appsec-generic-rules + ENROLL_INSTANCE_NAME: "pangolin-crowdsec" + PARSERS: crowdsecurity/whitelists + ACQUIRE_FILES: "/var/log/traefik/*.log" + ENROLL_TAGS: docker + healthcheck: + test: ["CMD", "cscli", "capi", "status"] + depends_on: + - gerbil # Wait for gerbil to be healthy + labels: + - "traefik.enable=false" # Disable traefik for crowdsec + volumes: + # crowdsec container data + - ./config/crowdsec:/etc/crowdsec # crowdsec config + - ./config/crowdsec/db:/var/lib/crowdsec/data # crowdsec db + # log bind mounts into crowdsec + - ./config/crowdsec_logs/auth.log:/var/log/auth.log:ro # auth.log + - ./config/crowdsec_logs/syslog:/var/log/syslog:ro # syslog + - ./config/crowdsec_logs:/var/log # crowdsec logs + - ./config/traefik/logs:/var/log/traefik # traefik logs + ports: + - 9090:9090 # port mapping for local firewall bouncers + - 6060:6060 # metrics endpoint for prometheus + expose: + - 9090 # http api for bouncers + - 6060 # metrics endpoint for prometheus + - 7422 # appsec waf endpoint + restart: unless-stopped + command: -t # Add test config flag to verify configuration \ No newline at end of file diff --git a/install/config/crowdsec/traefik_config.yml b/install/config/crowdsec/traefik_config.yml index 2ac9125c..59356ea7 100644 --- a/install/config/crowdsec/traefik_config.yml +++ b/install/config/crowdsec/traefik_config.yml @@ -80,7 +80,7 @@ entryPoints: http: tls: certResolver: "letsencrypt" - middlewares: # CHANGE MADE HERE (BOUNCER ENABLED) !!! + middlewares: - crowdsec@file serversTransport: diff --git a/install/crowdsec.go b/install/crowdsec.go index 1d9bef65..2b77985f 100644 --- a/install/crowdsec.go +++ b/install/crowdsec.go @@ -15,17 +15,40 @@ func installCrowdsec(config Config) error { return fmt.Errorf("backup failed: %v", err) } - if err := AddCrowdSecService("docker-compose.yml"); err != nil { - return fmt.Errorf("crowdsec service addition failed: %v", err) - } - if err := createConfigFiles(config); err != nil { fmt.Printf("Error creating config files: %v\n", err) os.Exit(1) } - // moveFile("config/crowdsec/traefik_config.yml", "config/traefik/traefik_config.yml") - moveFile("config/crowdsec/dynamic.yml", "config/traefik/dynamic.yml") + if err := copyDockerService("config/crowdsec/docker-compose.yml", "docker-compose.yml", "crowdsec"); err != nil { + fmt.Printf("Error copying docker service: %v\n", err) + os.Exit(1) + } + + if err := copyWebsecureEntryPoint("config/crowdsec/traefik_config.yml", "config/traefik/traefik_config.yml"); err != nil { + fmt.Printf("Error copying entry points: %v\n", err) + os.Exit(1) + } + + if err := copyEntryPoints("config/traefik/traefik_config.yml", "config/crowdsec/traefik_config.yml"); err != nil { + fmt.Printf("Error copying entry points: %v\n", err) + os.Exit(1) + } + + if err := moveFile("config/crowdsec/traefik_config.yml", "config/traefik/traefik_config.yml"); err != nil { + fmt.Printf("Error moving file: %v\n", err) + os.Exit(1) + } + + if err := moveFile("config/crowdsec/dynamic_config.yml", "config/traefik/dynamic_config.yml"); err != nil { + fmt.Printf("Error moving file: %v\n", err) + os.Exit(1) + } + + if err := os.Remove("config/crowdsec/docker-compose.yml"); err != nil { + fmt.Printf("Error removing file: %v\n", err) + os.Exit(1) + } if err := retrieveBouncerKey(config); err != nil { return fmt.Errorf("bouncer key retrieval failed: %v", err) From e6c42e9610feb7e656cc6c951fe84a0f3605c27d Mon Sep 17 00:00:00 2001 From: Owen Date: Sun, 16 Feb 2025 11:31:01 -0500 Subject: [PATCH 19/26] Indent 2 --- install/config.go | 43 ++++++++++++++++++++++++++++++++++++++++--- install/crowdsec.go | 19 ------------------- 2 files changed, 40 insertions(+), 22 deletions(-) diff --git a/install/config.go b/install/config.go index 1ebfbecb..b666b53e 100644 --- a/install/config.go +++ b/install/config.go @@ -4,6 +4,7 @@ import ( "bytes" "fmt" "os" + "os/exec" "gopkg.in/yaml.v3" ) @@ -139,7 +140,8 @@ func copyEntryPoints(sourceFile, destFile string) error { destYAML["entryPoints"] = entryPoints // Marshal updated destination YAML - updatedData, err := yaml.Marshal(destYAML) + // updatedData, err := yaml.Marshal(destYAML) + updatedData, err := MarshalYAMLWithIndent(destYAML, 2) if err != nil { return fmt.Errorf("error marshaling updated YAML: %w", err) } @@ -201,7 +203,8 @@ func copyWebsecureEntryPoint(sourceFile, destFile string) error { destEntryPoints["websecure"] = websecure // Marshal updated destination YAML - updatedData, err := yaml.Marshal(destYAML) + // updatedData, err := yaml.Marshal(destYAML) + updatedData, err := MarshalYAMLWithIndent(destYAML, 2) if err != nil { return fmt.Errorf("error marshaling updated YAML: %w", err) } @@ -264,7 +267,8 @@ func copyDockerService(sourceFile, destFile, serviceName string) error { // Marshal updated destination YAML // Use yaml.v3 encoder to preserve formatting and comments - updatedData, err := yaml.Marshal(destCompose) + // updatedData, err := yaml.Marshal(destCompose) + updatedData, err := MarshalYAMLWithIndent(destCompose, 2) if err != nil { return fmt.Errorf("error marshaling updated Docker Compose file: %w", err) } @@ -276,3 +280,36 @@ func copyDockerService(sourceFile, destFile, serviceName string) error { return nil } + +func backupConfig() error { + // Backup docker-compose.yml + if _, err := os.Stat("docker-compose.yml"); err == nil { + if err := copyFile("docker-compose.yml", "docker-compose.yml.backup"); err != nil { + return fmt.Errorf("failed to backup docker-compose.yml: %v", err) + } + } + + // Backup config directory + if _, err := os.Stat("config"); err == nil { + cmd := exec.Command("tar", "-czvf", "config.tar.gz", "config") + if err := cmd.Run(); err != nil { + return fmt.Errorf("failed to backup config directory: %v", err) + } + } + + return nil +} + +func MarshalYAMLWithIndent(data interface{}, indent int) ([]byte, error) { + buffer := new(bytes.Buffer) + encoder := yaml.NewEncoder(buffer) + encoder.SetIndent(indent) + + err := encoder.Encode(data) + if err != nil { + return nil, err + } + + defer encoder.Close() + return buffer.Bytes(), nil +} diff --git a/install/crowdsec.go b/install/crowdsec.go index 2b77985f..5b777e80 100644 --- a/install/crowdsec.go +++ b/install/crowdsec.go @@ -57,25 +57,6 @@ func installCrowdsec(config Config) error { return nil } -func backupConfig() error { - // Backup docker-compose.yml - if _, err := os.Stat("docker-compose.yml"); err == nil { - if err := copyFile("docker-compose.yml", "docker-compose.yml.backup"); err != nil { - return fmt.Errorf("failed to backup docker-compose.yml: %v", err) - } - } - - // Backup config directory - if _, err := os.Stat("config"); err == nil { - cmd := exec.Command("tar", "-czvf", "config.tar.gz", "config") - if err := cmd.Run(); err != nil { - return fmt.Errorf("failed to backup config directory: %v", err) - } - } - - return nil -} - func retrieveBouncerKey(config Config) error { // Start crowdsec container cmd := exec.Command("docker", "compose", "up", "-d", "crowdsec") From fd11fb81d6eb839c0ac401afd9877405e1824d83 Mon Sep 17 00:00:00 2001 From: Owen Date: Tue, 18 Feb 2025 21:41:23 -0500 Subject: [PATCH 20/26] Remove some config --- install/config.go | 20 +++++ install/config/crowdsec/config.yaml | 12 --- install/config/crowdsec/dynamic_config.yml | 4 +- .../crowdsec/local_api_credentials.yaml | 2 - install/crowdsec.go | 26 +++++- install/main.go | 79 +++++++++++++++++-- 6 files changed, 121 insertions(+), 22 deletions(-) delete mode 100644 install/config/crowdsec/config.yaml delete mode 100644 install/config/crowdsec/local_api_credentials.yaml diff --git a/install/config.go b/install/config.go index b666b53e..f87bb1af 100644 --- a/install/config.go +++ b/install/config.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "os/exec" + "strings" "gopkg.in/yaml.v3" ) @@ -313,3 +314,22 @@ func MarshalYAMLWithIndent(data interface{}, indent int) ([]byte, error) { defer encoder.Close() return buffer.Bytes(), nil } + +func replaceInFile(filepath, oldStr, newStr string) error { + // Read the file content + content, err := os.ReadFile(filepath) + if err != nil { + return fmt.Errorf("error reading file: %v", err) + } + + // Replace the string + newContent := strings.Replace(string(content), oldStr, newStr, -1) + + // Write the modified content back to the file + err = os.WriteFile(filepath, []byte(newContent), 0644) + if err != nil { + return fmt.Errorf("error writing file: %v", err) + } + + return nil +} diff --git a/install/config/crowdsec/config.yaml b/install/config/crowdsec/config.yaml deleted file mode 100644 index 0acf4635..00000000 --- a/install/config/crowdsec/config.yaml +++ /dev/null @@ -1,12 +0,0 @@ -api: - client: - insecure_skip_verify: false - credentials_path: /etc/crowdsec/local_api_credentials.yaml - server: - log_level: info - listen_uri: 0.0.0.0:9090 - profiles_path: /etc/crowdsec/profiles.yaml - trusted_ips: - - 0.0.0.0/0 - - 127.0.0.1 - - ::1 \ No newline at end of file diff --git a/install/config/crowdsec/dynamic_config.yml b/install/config/crowdsec/dynamic_config.yml index d2556971..a3d32dbd 100644 --- a/install/config/crowdsec/dynamic_config.yml +++ b/install/config/crowdsec/dynamic_config.yml @@ -42,8 +42,8 @@ http: crowdsecAppsecHost: crowdsec:7422 # CrowdSec IP address which you noted down later crowdsecAppsecFailureBlock: true # Block on failure crowdsecAppsecUnreachableBlock: true # Block on unreachable - crowdsecLapiKey: "{{.TraefikBouncerKey}}" # CrowdSec API key which you noted down later - crowdsecLapiHost: crowdsec:9090 # CrowdSec + crowdsecLapiKey: "PUT_YOUR_BOUNCER_KEY_HERE_OR_IT_WILL_NOT_WORK" # CrowdSec API key which you noted down later + crowdsecLapiHost: crowdsec:8080 # CrowdSec crowdsecLapiScheme: http # CrowdSec API scheme forwardedHeadersTrustedIPs: # Forwarded headers trusted IPs - "0.0.0.0/0" # All IP addresses are trusted for forwarded headers (CHANGE MADE HERE) diff --git a/install/config/crowdsec/local_api_credentials.yaml b/install/config/crowdsec/local_api_credentials.yaml deleted file mode 100644 index 8776e4fd..00000000 --- a/install/config/crowdsec/local_api_credentials.yaml +++ /dev/null @@ -1,2 +0,0 @@ -url: http://0.0.0.0:9090 -login: localhost \ No newline at end of file diff --git a/install/crowdsec.go b/install/crowdsec.go index 5b777e80..6d25a633 100644 --- a/install/crowdsec.go +++ b/install/crowdsec.go @@ -10,6 +10,11 @@ import ( ) func installCrowdsec(config Config) error { + + if err := stopContainers(); err != nil { + return fmt.Errorf("failed to stop containers: %v", err) + } + // Run installation steps if err := backupConfig(); err != nil { return fmt.Errorf("backup failed: %v", err) @@ -20,6 +25,10 @@ func installCrowdsec(config Config) error { os.Exit(1) } + os.MkdirAll("config/crowdsec/db", 0755) + os.MkdirAll("config/crowdsec_logs/syslog", 0755) + os.MkdirAll("config/traefik/logs", 0755) + if err := copyDockerService("config/crowdsec/docker-compose.yml", "docker-compose.yml", "crowdsec"); err != nil { fmt.Printf("Error copying docker service: %v\n", err) os.Exit(1) @@ -54,16 +63,22 @@ func installCrowdsec(config Config) error { return fmt.Errorf("bouncer key retrieval failed: %v", err) } + // if err := startContainers(); err != nil { + // return fmt.Errorf("failed to start containers: %v", err) + // } + return nil } func retrieveBouncerKey(config Config) error { + + fmt.Println("Retrieving bouncer key. Please be patient...") + // Start crowdsec container cmd := exec.Command("docker", "compose", "up", "-d", "crowdsec") if err := cmd.Run(); err != nil { return fmt.Errorf("failed to start crowdsec: %v", err) } - defer exec.Command("docker", "compose", "down").Run() // verify that the container is running if not keep waiting for 10 more seconds then return an error count := 0 @@ -95,10 +110,19 @@ func retrieveBouncerKey(config Config) error { for _, line := range lines { if strings.Contains(line, "key:") { config.TraefikBouncerKey = strings.TrimSpace(strings.Split(line, ":")[1]) + fmt.Println("Bouncer key:", config.TraefikBouncerKey) break } } + // Stop crowdsec container + cmd = exec.Command("docker", "compose", "down") + if err := cmd.Run(); err != nil { + return fmt.Errorf("failed to stop crowdsec: %v", err) + } + + fmt.Println("Bouncer key retrieved successfully.") + return nil } diff --git a/install/main.go b/install/main.go index e5ad98b9..bca19100 100644 --- a/install/main.go +++ b/install/main.go @@ -179,11 +179,6 @@ func readInt(reader *bufio.Reader, prompt string, defaultValue int) int { return value } -func isDockerFilePresent() bool { - _, err := os.Stat("docker-compose.yml") - return !os.IsNotExist(err) -} - func collectUserInput(reader *bufio.Reader) Config { config := Config{} @@ -521,6 +516,80 @@ func pullAndStartContainers() error { return nil } +// bring containers down +func stopContainers() error { + fmt.Println("Stopping containers...") + + // Check which docker compose command is available + var useNewStyle bool + checkCmd := exec.Command("docker", "compose", "version") + if err := checkCmd.Run(); err == nil { + useNewStyle = true + } else { + // Check if docker-compose (old style) is available + checkCmd = exec.Command("docker-compose", "version") + if err := checkCmd.Run(); err != nil { + return fmt.Errorf("neither 'docker compose' nor 'docker-compose' command is available: %v", err) + } + } + + // Helper function to execute docker compose commands + executeCommand := func(args ...string) error { + var cmd *exec.Cmd + if useNewStyle { + cmd = exec.Command("docker", append([]string{"compose"}, args...)...) + } else { + cmd = exec.Command("docker-compose", args...) + } + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd.Run() + } + + if err := executeCommand("-f", "docker-compose.yml", "down"); err != nil { + return fmt.Errorf("failed to stop containers: %v", err) + } + + return nil +} + +// just start containers +func startContainers() error { + fmt.Println("Starting containers...") + + // Check which docker compose command is available + var useNewStyle bool + checkCmd := exec.Command("docker", "compose", "version") + if err := checkCmd.Run(); err == nil { + useNewStyle = true + } else { + // Check if docker-compose (old style) is available + checkCmd = exec.Command("docker-compose", "version") + if err := checkCmd.Run(); err != nil { + return fmt.Errorf("neither 'docker compose' nor 'docker-compose' command is available: %v", err) + } + } + + // Helper function to execute docker compose commands + executeCommand := func(args ...string) error { + var cmd *exec.Cmd + if useNewStyle { + cmd = exec.Command("docker", append([]string{"compose"}, args...)...) + } else { + cmd = exec.Command("docker-compose", args...) + } + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd.Run() + } + + if err := executeCommand("-f", "docker-compose.yml", "up", "-d"); err != nil { + return fmt.Errorf("failed to start containers: %v", err) + } + + return nil +} + func copyFile(src, dst string) error { source, err := os.Open(src) if err != nil { From 3194dc56eb990cac131a070c148538cc68d944e2 Mon Sep 17 00:00:00 2001 From: Owen Date: Wed, 19 Feb 2025 09:44:40 -0500 Subject: [PATCH 21/26] Move path to first in dropdown --- .../settings/resources/[resourceId]/rules/page.tsx | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/app/[orgId]/settings/resources/[resourceId]/rules/page.tsx b/src/app/[orgId]/settings/resources/[resourceId]/rules/page.tsx index 1b9eb6ca..a3fb033d 100644 --- a/src/app/[orgId]/settings/resources/[resourceId]/rules/page.tsx +++ b/src/app/[orgId]/settings/resources/[resourceId]/rules/page.tsx @@ -92,9 +92,9 @@ enum RuleAction { } enum RuleMatch { + PATH = "Path", IP = "IP", CIDR = "IP Range", - PATH = "Path" } export default function ResourceRules(props: { @@ -469,9 +469,9 @@ export default function ResourceRules(props: { + {RuleMatch.PATH} {RuleMatch.IP} {RuleMatch.CIDR} - {RuleMatch.PATH} ) @@ -665,17 +665,17 @@ export default function ResourceRules(props: { + {resource.http && ( + + {RuleMatch.PATH} + + )} {RuleMatch.IP} {RuleMatch.CIDR} - {resource.http && ( - - {RuleMatch.PATH} - - )} From 5f95500b6f77d3da6dfb27cc189217adc7d0a884 Mon Sep 17 00:00:00 2001 From: Owen Date: Wed, 19 Feb 2025 21:42:42 -0500 Subject: [PATCH 22/26] Crowdsec installer works? --- install/config.go | 58 +++++++++++++++++++ install/config/docker-compose.yml | 1 + install/crowdsec.go | 95 +++++++++++++------------------ install/main.go | 68 +++++++++++++++++++++- 4 files changed, 165 insertions(+), 57 deletions(-) diff --git a/install/config.go b/install/config.go index f87bb1af..c31149d6 100644 --- a/install/config.go +++ b/install/config.go @@ -333,3 +333,61 @@ func replaceInFile(filepath, oldStr, newStr string) error { return nil } + +func CheckAndAddTraefikLogVolume(composePath string) error { + // Read the docker-compose.yml file + data, err := os.ReadFile(composePath) + if err != nil { + return fmt.Errorf("error reading compose file: %w", err) + } + + // Parse YAML into a generic map + var compose map[string]interface{} + if err := yaml.Unmarshal(data, &compose); err != nil { + return fmt.Errorf("error parsing compose file: %w", err) + } + + // Get services section + services, ok := compose["services"].(map[string]interface{}) + if !ok { + return fmt.Errorf("services section not found or invalid") + } + + // Get traefik service + traefik, ok := services["traefik"].(map[string]interface{}) + if !ok { + return fmt.Errorf("traefik service not found or invalid") + } + + // Check volumes + logVolume := "./config/traefik/logs:/var/log/traefik" + var volumes []interface{} + + if existingVolumes, ok := traefik["volumes"].([]interface{}); ok { + // Check if volume already exists + for _, v := range existingVolumes { + if v.(string) == logVolume { + fmt.Println("Traefik log volume is already configured") + return nil + } + } + volumes = existingVolumes + } + + // Add new volume + volumes = append(volumes, logVolume) + traefik["volumes"] = volumes + + // Write updated config back to file + newData, err := MarshalYAMLWithIndent(compose, 2) + if err != nil { + return fmt.Errorf("error marshaling updated compose file: %w", err) + } + + if err := os.WriteFile(composePath, newData, 0644); err != nil { + return fmt.Errorf("error writing updated compose file: %w", err) + } + + fmt.Println("Added traefik log volume and created logs directory") + return nil +} diff --git a/install/config/docker-compose.yml b/install/config/docker-compose.yml index 42604ab4..f6ce7892 100644 --- a/install/config/docker-compose.yml +++ b/install/config/docker-compose.yml @@ -52,6 +52,7 @@ services: volumes: - ./config/traefik:/etc/traefik:ro # Volume to store the Traefik configuration - ./config/letsencrypt:/letsencrypt # Volume to store the Let's Encrypt certificates + - ./config/traefik/logs:/var/log/traefik # Volume to store Traefik logs networks: default: diff --git a/install/crowdsec.go b/install/crowdsec.go index 6d25a633..d98b2acf 100644 --- a/install/crowdsec.go +++ b/install/crowdsec.go @@ -6,7 +6,6 @@ import ( "os" "os/exec" "strings" - "time" ) func installCrowdsec(config Config) error { @@ -59,70 +58,30 @@ func installCrowdsec(config Config) error { os.Exit(1) } - if err := retrieveBouncerKey(config); err != nil { - return fmt.Errorf("bouncer key retrieval failed: %v", err) + if err := CheckAndAddTraefikLogVolume("docker-compose.yml"); err != nil { + fmt.Printf("Error checking and adding Traefik log volume: %v\n", err) + os.Exit(1) } - // if err := startContainers(); err != nil { - // return fmt.Errorf("failed to start containers: %v", err) - // } - - return nil -} - -func retrieveBouncerKey(config Config) error { - - fmt.Println("Retrieving bouncer key. Please be patient...") - - // Start crowdsec container - cmd := exec.Command("docker", "compose", "up", "-d", "crowdsec") - if err := cmd.Run(); err != nil { - return fmt.Errorf("failed to start crowdsec: %v", err) + if err := startContainers(); err != nil { + return fmt.Errorf("failed to start containers: %v", err) } - // verify that the container is running if not keep waiting for 10 more seconds then return an error - count := 0 - for { - cmd := exec.Command("docker", "inspect", "-f", "{{.State.Running}}", "crowdsec") - output, err := cmd.Output() - if err != nil { - return fmt.Errorf("failed to inspect crowdsec container: %v", err) - } - if strings.TrimSpace(string(output)) == "true" { - break - } - time.Sleep(10 * time.Second) - count++ - - if count > 4 { - return fmt.Errorf("crowdsec container is not running") - } - } - - // Get bouncer key - output, err := exec.Command("docker", "exec", "crowdsec", "cscli", "bouncers", "add", "traefik-bouncer").Output() + // get API key + apiKey, err := GetCrowdSecAPIKey() if err != nil { - return fmt.Errorf("failed to get bouncer key: %v", err) + return fmt.Errorf("failed to get API key: %v", err) + } + config.TraefikBouncerKey = apiKey + + if err := replaceInFile("config/traefik/dynamic_config.yml", "PUT_YOUR_BOUNCER_KEY_HERE_OR_IT_WILL_NOT_WORK", config.TraefikBouncerKey); err != nil { + return fmt.Errorf("failed to replace bouncer key: %v", err) } - // Parse key from output - lines := strings.Split(string(output), "\n") - for _, line := range lines { - if strings.Contains(line, "key:") { - config.TraefikBouncerKey = strings.TrimSpace(strings.Split(line, ":")[1]) - fmt.Println("Bouncer key:", config.TraefikBouncerKey) - break - } + if err := restartContainer("traefik"); err != nil { + return fmt.Errorf("failed to restart containers: %v", err) } - // Stop crowdsec container - cmd = exec.Command("docker", "compose", "down") - if err := cmd.Run(); err != nil { - return fmt.Errorf("failed to stop crowdsec: %v", err) - } - - fmt.Println("Bouncer key retrieved successfully.") - return nil } @@ -136,3 +95,27 @@ func checkIsCrowdsecInstalledInCompose() bool { // Check for crowdsec service return bytes.Contains(content, []byte("crowdsec:")) } + +func GetCrowdSecAPIKey() (string, error) { + // First, ensure the container is running + if err := waitForContainer("crowdsec"); err != nil { + return "", fmt.Errorf("waiting for container: %w", err) + } + + // Execute the command to get the API key + cmd := exec.Command("docker", "exec", "crowdsec", "cscli", "bouncers", "add", "traefik-bouncer", "-o", "raw") + var out bytes.Buffer + cmd.Stdout = &out + + if err := cmd.Run(); err != nil { + return "", fmt.Errorf("executing command: %w", err) + } + + // Trim any whitespace from the output + apiKey := strings.TrimSpace(out.String()) + if apiKey == "" { + return "", fmt.Errorf("empty API key returned") + } + + return apiKey, nil +} diff --git a/install/main.go b/install/main.go index bca19100..9064b4f7 100644 --- a/install/main.go +++ b/install/main.go @@ -4,14 +4,16 @@ import ( "bufio" "embed" "fmt" + "io" "io/fs" "os" - "io" + "time" "os/exec" "path/filepath" "runtime" "strings" "syscall" + "bytes" "text/template" "unicode" @@ -590,6 +592,42 @@ func startContainers() error { return nil } +func restartContainer(container string) error { + fmt.Printf("Restarting %s container...\n", container) + + // Check which docker compose command is available + var useNewStyle bool + checkCmd := exec.Command("docker", "compose", "version") + if err := checkCmd.Run(); err == nil { + useNewStyle = true + } else { + // Check if docker-compose (old style) is available + checkCmd = exec.Command("docker-compose", "version") + if err := checkCmd.Run(); err != nil { + return fmt.Errorf("neither 'docker compose' nor 'docker-compose' command is available: %v", err) + } + } + + // Helper function to execute docker compose commands + executeCommand := func(args ...string) error { + var cmd *exec.Cmd + if useNewStyle { + cmd = exec.Command("docker", append([]string{"compose"}, args...)...) + } else { + cmd = exec.Command("docker-compose", args...) + } + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd.Run() + } + + if err := executeCommand("-f", "docker-compose.yml", "restart", container); err != nil { + return fmt.Errorf("failed to restart %s container: %v", container, err) + } + + return nil +} + func copyFile(src, dst string) error { source, err := os.Open(src) if err != nil { @@ -613,4 +651,32 @@ func moveFile(src, dst string) error { } return os.Remove(src) +} + +func waitForContainer(containerName string) error { + maxAttempts := 30 + retryInterval := time.Second * 2 + + for attempt := 0; attempt < maxAttempts; attempt++ { + // Check if container is running + cmd := exec.Command("docker", "container", "inspect", "-f", "{{.State.Running}}", containerName) + var out bytes.Buffer + cmd.Stdout = &out + + if err := cmd.Run(); err != nil { + // If the container doesn't exist or there's another error, wait and retry + time.Sleep(retryInterval) + continue + } + + isRunning := strings.TrimSpace(out.String()) == "true" + if isRunning { + return nil + } + + // Container exists but isn't running yet, wait and retry + time.Sleep(retryInterval) + } + + return fmt.Errorf("container %s did not start within %v seconds", containerName, maxAttempts*int(retryInterval.Seconds())) } \ No newline at end of file From 372932985db67047dba5aef96e440e77a9b54b52 Mon Sep 17 00:00:00 2001 From: Milo Schwartz Date: Sun, 23 Feb 2025 17:34:28 -0500 Subject: [PATCH 23/26] update README.md --- README.md | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 61e1a689..324680f5 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,13 @@ +

Tunneled Mesh Reverse Proxy Server with Access Control

+
+ +_Your own self-hosted zero trust tunnel._ + +
+ -

Tunneled Mesh Reverse Proxy Server with Access Control

-
- -_Your own self-hosted zero trust tunnel._ - -
- Pangolin is a self-hosted tunneled reverse proxy server with identity and access control, designed to securely expose private resources on distributed networks. Acting as a central hub, it connects isolated networks — even those behind restrictive firewalls — through encrypted tunnels, enabling easy access to remote services without opening ports. Preview @@ -108,7 +108,11 @@ _Sites page of Pangolin dashboard (dark mode) showing multiple tunnels connected 1. **Deploy the Central Server**: - - Deploy the Docker Compose stack onto a VPS hosted on a cloud platform like Amazon EC2, DigitalOcean Droplet, or similar. There are many cheap VPS hosting options available to suit your needs. + - Deploy the Docker Compose stack onto a VPS hosted on a cloud platform like RackNerd, Amazon EC2, DigitalOcean Droplet, or similar. There are many cheap VPS hosting options available to suit your needs. + +> [!TIP] +> Many of our users have had a great experience with [RackNerd](https://my.racknerd.com/aff.php?aff=13788). Depending on promotions, you can likely get a **VPS with 1 vCPU, 1GB RAM, and ~20GB SSD for just around $12/year**. That's a great deal! +> We are part of the [RackNerd](https://my.racknerd.com/aff.php?aff=13788) affiliate program, so if you sign up using [our link](https://my.racknerd.com/aff.php?aff=13788), we receive a small commission which helps us maintain the project and keep it free for everyone. 2. **Domain Configuration**: From f59f0ee57dc66e83be73127b626975a1eab8cf29 Mon Sep 17 00:00:00 2001 From: Owen Date: Sun, 23 Feb 2025 21:44:02 -0500 Subject: [PATCH 24/26] Merge yaml files instead? --- install/config.go | 184 +++++++++++++++++--------------------------- install/crowdsec.go | 20 ++--- 2 files changed, 82 insertions(+), 122 deletions(-) diff --git a/install/config.go b/install/config.go index c31149d6..3be62601 100644 --- a/install/config.go +++ b/install/config.go @@ -106,118 +106,6 @@ func findPattern(s, pattern string) int { return bytes.Index([]byte(s), []byte(pattern)) } -func copyEntryPoints(sourceFile, destFile string) error { - // Read source file - sourceData, err := os.ReadFile(sourceFile) - if err != nil { - return fmt.Errorf("error reading source file: %w", err) - } - - // Read destination file - destData, err := os.ReadFile(destFile) - if err != nil { - return fmt.Errorf("error reading destination file: %w", err) - } - - // Parse source YAML - var sourceYAML map[string]interface{} - if err := yaml.Unmarshal(sourceData, &sourceYAML); err != nil { - return fmt.Errorf("error parsing source YAML: %w", err) - } - - // Parse destination YAML - var destYAML map[string]interface{} - if err := yaml.Unmarshal(destData, &destYAML); err != nil { - return fmt.Errorf("error parsing destination YAML: %w", err) - } - - // Get entryPoints section from source - entryPoints, ok := sourceYAML["entryPoints"] - if !ok { - return fmt.Errorf("entryPoints section not found in source file") - } - - // Update entryPoints in destination - destYAML["entryPoints"] = entryPoints - - // Marshal updated destination YAML - // updatedData, err := yaml.Marshal(destYAML) - updatedData, err := MarshalYAMLWithIndent(destYAML, 2) - if err != nil { - return fmt.Errorf("error marshaling updated YAML: %w", err) - } - - // Write updated YAML back to destination file - if err := os.WriteFile(destFile, updatedData, 0644); err != nil { - return fmt.Errorf("error writing to destination file: %w", err) - } - - return nil -} - -func copyWebsecureEntryPoint(sourceFile, destFile string) error { - // Read source file - sourceData, err := os.ReadFile(sourceFile) - if err != nil { - return fmt.Errorf("error reading source file: %w", err) - } - - // Read destination file - destData, err := os.ReadFile(destFile) - if err != nil { - return fmt.Errorf("error reading destination file: %w", err) - } - - // Parse source YAML - var sourceYAML map[string]interface{} - if err := yaml.Unmarshal(sourceData, &sourceYAML); err != nil { - return fmt.Errorf("error parsing source YAML: %w", err) - } - - // Parse destination YAML - var destYAML map[string]interface{} - if err := yaml.Unmarshal(destData, &destYAML); err != nil { - return fmt.Errorf("error parsing destination YAML: %w", err) - } - - // Get entryPoints section from source - entryPoints, ok := sourceYAML["entryPoints"].(map[string]interface{}) - if !ok { - return fmt.Errorf("entryPoints section not found in source file or has invalid format") - } - - // Get websecure configuration - websecure, ok := entryPoints["websecure"] - if !ok { - return fmt.Errorf("websecure entrypoint not found in source file") - } - - // Get or create entryPoints section in destination - destEntryPoints, ok := destYAML["entryPoints"].(map[string]interface{}) - if !ok { - // If entryPoints section doesn't exist, create it - destEntryPoints = make(map[string]interface{}) - destYAML["entryPoints"] = destEntryPoints - } - - // Update websecure in destination - destEntryPoints["websecure"] = websecure - - // Marshal updated destination YAML - // updatedData, err := yaml.Marshal(destYAML) - updatedData, err := MarshalYAMLWithIndent(destYAML, 2) - if err != nil { - return fmt.Errorf("error marshaling updated YAML: %w", err) - } - - // Write updated YAML back to destination file - if err := os.WriteFile(destFile, updatedData, 0644); err != nil { - return fmt.Errorf("error writing to destination file: %w", err) - } - - return nil -} - func copyDockerService(sourceFile, destFile, serviceName string) error { // Read source file sourceData, err := os.ReadFile(sourceFile) @@ -391,3 +279,75 @@ func CheckAndAddTraefikLogVolume(composePath string) error { fmt.Println("Added traefik log volume and created logs directory") return nil } + +// MergeYAML merges two YAML files, where the contents of the second file +// are merged into the first file. In case of conflicts, values from the +// second file take precedence. +func MergeYAML(baseFile, overlayFile string) error { + // Read the base YAML file + baseContent, err := os.ReadFile(baseFile) + if err != nil { + return fmt.Errorf("error reading base file: %v", err) + } + + // Read the overlay YAML file + overlayContent, err := os.ReadFile(overlayFile) + if err != nil { + return fmt.Errorf("error reading overlay file: %v", err) + } + + // Parse base YAML into a map + var baseMap map[string]interface{} + if err := yaml.Unmarshal(baseContent, &baseMap); err != nil { + return fmt.Errorf("error parsing base YAML: %v", err) + } + + // Parse overlay YAML into a map + var overlayMap map[string]interface{} + if err := yaml.Unmarshal(overlayContent, &overlayMap); err != nil { + return fmt.Errorf("error parsing overlay YAML: %v", err) + } + + // Merge the overlay into the base + merged := mergeMap(baseMap, overlayMap) + + // Marshal the merged result back to YAML + mergedContent, err := MarshalYAMLWithIndent(merged, 2) + if err != nil { + return fmt.Errorf("error marshaling merged YAML: %v", err) + } + + // Write the merged content back to the base file + if err := os.WriteFile(baseFile, mergedContent, 0644); err != nil { + return fmt.Errorf("error writing merged YAML: %v", err) + } + + return nil +} + +// mergeMap recursively merges two maps +func mergeMap(base, overlay map[string]interface{}) map[string]interface{} { + result := make(map[string]interface{}) + + // Copy all key-values from base map + for k, v := range base { + result[k] = v + } + + // Merge overlay values + for k, v := range overlay { + // If both maps have the same key and both values are maps, merge recursively + if baseVal, ok := base[k]; ok { + if baseMap, isBaseMap := baseVal.(map[string]interface{}); isBaseMap { + if overlayMap, isOverlayMap := v.(map[string]interface{}); isOverlayMap { + result[k] = mergeMap(baseMap, overlayMap) + continue + } + } + } + // Otherwise, overlay value takes precedence + result[k] = v + } + + return result +} diff --git a/install/crowdsec.go b/install/crowdsec.go index d98b2acf..2d56ecc6 100644 --- a/install/crowdsec.go +++ b/install/crowdsec.go @@ -33,23 +33,23 @@ func installCrowdsec(config Config) error { os.Exit(1) } - if err := copyWebsecureEntryPoint("config/crowdsec/traefik_config.yml", "config/traefik/traefik_config.yml"); err != nil { + if err := MergeYAML("config/traefik/traefik_config.yml", "config/crowdsec/traefik_config.yml"); err != nil { fmt.Printf("Error copying entry points: %v\n", err) os.Exit(1) } + // delete the 2nd file + if err := os.Remove("config/crowdsec/traefik_config.yml"); err != nil { + fmt.Printf("Error removing file: %v\n", err) + os.Exit(1) + } - if err := copyEntryPoints("config/traefik/traefik_config.yml", "config/crowdsec/traefik_config.yml"); err != nil { + if err := MergeYAML("config/traefik/dynamic_config.yml", "config/crowdsec/dynamic_config.yml"); err != nil { fmt.Printf("Error copying entry points: %v\n", err) os.Exit(1) } - - if err := moveFile("config/crowdsec/traefik_config.yml", "config/traefik/traefik_config.yml"); err != nil { - fmt.Printf("Error moving file: %v\n", err) - os.Exit(1) - } - - if err := moveFile("config/crowdsec/dynamic_config.yml", "config/traefik/dynamic_config.yml"); err != nil { - fmt.Printf("Error moving file: %v\n", err) + // delete the 2nd file + if err := os.Remove("config/crowdsec/dynamic_config.yml"); err != nil { + fmt.Printf("Error removing file: %v\n", err) os.Exit(1) } From ccbe56e110460c93a6507c4b177a26f731d94fcc Mon Sep 17 00:00:00 2001 From: Milo Schwartz Date: Mon, 24 Feb 2025 12:03:48 -0500 Subject: [PATCH 25/26] update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 324680f5..5baef277 100644 --- a/README.md +++ b/README.md @@ -151,7 +151,7 @@ View the [project board](https://github.com/orgs/fosrl/projects/1) for more deta ## Licensing -Pangolin is dual licensed under the AGPLv3 and the Fossorial Commercial license. For inquiries about commercial licensing, please contact us. +Pangolin is dual licensed under the AGPLv3 and the Fossorial Commercial license. For inquiries about commercial licensing, please contact us at [numbat@fossorial.io](mailto:numbat@fossorial.io). ## Contributions From 06c434a5eaefe81f9f17f8b57e80ef52ca95d761 Mon Sep 17 00:00:00 2001 From: Owen Date: Wed, 26 Feb 2025 21:13:52 -0500 Subject: [PATCH 26/26] Copy in the right versions when building --- install/Makefile | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/install/Makefile b/install/Makefile index e8e9cd2e..9bde02cf 100644 --- a/install/Makefile +++ b/install/Makefile @@ -1,13 +1,24 @@ -all: build +all: update-versions go-build-release put-back -build: - CGO_ENABLED=0 go build -o bin/installer - -release: +go-build-release: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o bin/installer_linux_amd64 CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -o bin/installer_linux_arm64 clean: - rm -f bin/installer rm -f bin/installer_linux_amd64 rm -f bin/installer_linux_arm64 + +update-versions: + @echo "Fetching latest versions..." + cp main.go main.go.bak && \ + PANGOLIN_VERSION=$$(curl -s https://api.github.com/repos/fosrl/pangolin/tags | jq -r '.[0].name') && \ + GERBIL_VERSION=$$(curl -s https://api.github.com/repos/fosrl/gerbil/tags | jq -r '.[0].name') && \ + BADGER_VERSION=$$(curl -s https://api.github.com/repos/fosrl/badger/tags | jq -r '.[0].name') && \ + echo "Latest versions - Pangolin: $$PANGOLIN_VERSION, Gerbil: $$GERBIL_VERSION, Badger: $$BADGER_VERSION" && \ + sed -i "s/config.PangolinVersion = \".*\"/config.PangolinVersion = \"$$PANGOLIN_VERSION\"/" main.go && \ + sed -i "s/config.GerbilVersion = \".*\"/config.GerbilVersion = \"$$GERBIL_VERSION\"/" main.go && \ + sed -i "s/config.BadgerVersion = \".*\"/config.BadgerVersion = \"$$BADGER_VERSION\"/" main.go && \ + echo "Updated main.go with latest versions" + +put-back: + mv main.go.bak main.go \ No newline at end of file