From 6e2fbe9c7d5c5a69e6a0ee4f2331f6776593081c Mon Sep 17 00:00:00 2001 From: Jess Lowe Date: Wed, 19 Aug 2026 00:26:19 +0000 Subject: [PATCH 1/7] debian copyright mirror, but in go --- .../debian-copyright-mirror/Dockerfile | 18 +- .../mirrors/debian-copyright-mirror/main.go | 415 ++++++++++++++++++ .../debian-copyright-mirror/main_test.go | 193 ++++++++ vulnfeeds/gcs-tools/gcs.go | 13 + vulnfeeds/gcs-tools/gcs_test.go | 47 ++ vulnfeeds/utility/archive.go | 118 +++++ vulnfeeds/utility/archive_test.go | 69 +++ vulnfeeds/utility/decompress.go | 127 ++++++ vulnfeeds/utility/decompress_test.go | 108 +++++ vulnfeeds/utility/download.go | 102 +++++ vulnfeeds/utility/download_test.go | 66 +++ 11 files changed, 1273 insertions(+), 3 deletions(-) create mode 100644 vulnfeeds/cmd/mirrors/debian-copyright-mirror/main.go create mode 100644 vulnfeeds/cmd/mirrors/debian-copyright-mirror/main_test.go create mode 100644 vulnfeeds/utility/archive.go create mode 100644 vulnfeeds/utility/archive_test.go create mode 100644 vulnfeeds/utility/decompress.go create mode 100644 vulnfeeds/utility/decompress_test.go create mode 100644 vulnfeeds/utility/download.go create mode 100644 vulnfeeds/utility/download_test.go diff --git a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/Dockerfile b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/Dockerfile index 09afd30ba3f..6ef0187b35d 100644 --- a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/Dockerfile +++ b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/Dockerfile @@ -12,11 +12,23 @@ # See the License for the specific language governing permissions and # limitations under the License. +FROM golang:1.26.5-alpine@sha256:0178a641fbb4858c5f1b48e34bdaabe0350a330a1b1149aabd498d0699ff5fb2 AS GO_BUILD + +RUN mkdir /src +WORKDIR /src + +COPY ./go.mod /src/go.mod +COPY ./go.sum /src/go.sum +RUN go mod download + +COPY ./ /src/ +RUN CGO_ENABLED=0 go build -o debian-copyright-mirror ./cmd/mirrors/debian-copyright-mirror + FROM gcr.io/google.com/cloudsdktool/google-cloud-cli:alpine@sha256:be40864452bd6d7be21632a1dc18adf03d423de0bf2595f4a287c22400c484bb -RUN apk add py3-yaml +RUN apk add --no-cache xz curl -COPY ./debian-copyright-mirror.sh / -COPY ./debian-copyright-mirror.py / +COPY --from=GO_BUILD /src/debian-copyright-mirror /usr/local/bin/ +COPY ./cmd/mirrors/debian-copyright-mirror/debian-copyright-mirror.sh / ENTRYPOINT ["/debian-copyright-mirror.sh"] diff --git a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main.go b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main.go new file mode 100644 index 00000000000..19b314650f1 --- /dev/null +++ b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main.go @@ -0,0 +1,415 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package main downloads all machine-readable copyright files for packages in Debian unstable. +package main + +import ( + "bufio" + "context" + "flag" + "fmt" + "io" + "log/slog" + "net/http" + "os" + "os/exec" + "path/filepath" + "strings" + "sync/atomic" + "time" + + "cloud.google.com/go/storage" + gcs "github.com/google/osv.dev/vulnfeeds/gcs-tools" + "github.com/google/osv.dev/vulnfeeds/utility" + "github.com/google/osv.dev/vulnfeeds/utility/logger" + "golang.org/x/sync/errgroup" +) + +const ( + DefaultFilelistURL = "https://metadata.ftp-master.debian.org/changelogs/filelist.yaml.xz" + DefaultURLBase = "https://metadata.ftp-master.debian.org/changelogs" + DefaultPrefixFilter = "main/" + DefaultMinExpectedFiles = 40000 + DefaultNumWorkers = 50 + DefaultMaxFailureRate = 0.05 // 5% maximum allowed failure rate +) + +// ExtractUnstableCopyright parses Debian changelogs filelist YAML content from a reader +// and extracts the unstable_copyright path for each package matching the specified prefixFilter. +func ExtractUnstableCopyright(r io.Reader, prefixFilter string) ([]string, error) { + scanner := bufio.NewScanner(r) + const maxLineLen = 1024 * 1024 + buf := make([]byte, 64*1024) + scanner.Buffer(buf, maxLineLen) + + var results []string + inUnstable := false + currentPkgFound := false + + for scanner.Scan() { + line := scanner.Text() + // Top-level package key: starts without whitespace and ends with colon + if len(line) > 0 && line[0] != ' ' && line[0] != '\t' && strings.HasSuffix(line, ":") { + inUnstable = false + currentPkgFound = false + continue + } + + trimmed := strings.TrimSpace(line) + if trimmed == "" || strings.HasPrefix(trimmed, "#") { + continue + } + + // Check if this is a suite/version subkey under the package + if strings.HasPrefix(line, " ") && !strings.HasPrefix(line, " -") && strings.HasSuffix(line, ":") { + if trimmed == "unstable:" || trimmed == "'unstable':" || trimmed == `"unstable":` { + inUnstable = true + } else { + inUnstable = false + } + continue + } + + // If we are within the "unstable" section and haven't found a copyright file for this package yet + if inUnstable && !currentPkgFound && strings.HasPrefix(trimmed, "- ") { + entry := strings.TrimSpace(strings.TrimPrefix(trimmed, "- ")) + entry = strings.Trim(entry, `"'`) + if strings.HasSuffix(entry, "unstable_copyright") { + if prefixFilter == "" || strings.HasPrefix(entry, prefixFilter) { + results = append(results, entry) + } + currentPkgFound = true + } + } + } + + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("error scanning filelist YAML: %w", err) + } + + return results, nil +} + +// DownloadFilesConcurrently downloads all files in filelist using a channel-based worker pool. +func DownloadFilesConcurrently(ctx context.Context, client *http.Client, filelist []string, urlBase, workDir string, numWorkers int, skipExisting bool, maxFailureRate float64) error { + if numWorkers <= 0 { + numWorkers = DefaultNumWorkers + } + + total := len(filelist) + logger.Info("Starting concurrent download of copyright files", + slog.Int("total_files", total), + slog.Int("workers", numWorkers), + slog.Bool("skip_existing", skipExisting), + ) + + jobs := make(chan string, numWorkers*2) + var completed atomic.Int64 + var failed atomic.Int64 + var skipped atomic.Int64 + startTime := time.Now() + + g, ctx := errgroup.WithContext(ctx) + + for i := 0; i < numWorkers; i++ { + g.Go(func() error { + for path := range jobs { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + fullURL := fmt.Sprintf("%s/%s", strings.TrimRight(urlBase, "/"), strings.TrimLeft(path, "/")) + destPath := filepath.Join(workDir, path) + + if skipExisting { + if fi, err := os.Stat(destPath); err == nil && fi.Size() > 0 { + skipped.Add(1) + count := completed.Add(1) + logProgress(count, total, startTime) + continue + } + } + + if err := utility.DownloadFile(ctx, client, fullURL, destPath, false); err != nil { + failed.Add(1) + logger.Warn("Failed to download copyright file", slog.String("url", fullURL), slog.Any("err", err)) + } else { + count := completed.Add(1) + logProgress(count, total, startTime) + } + } + return nil + }) + } + + go func() { + defer close(jobs) + for _, path := range filelist { + select { + case <-ctx.Done(): + return + case jobs <- path: + } + } + }() + + if err := g.Wait(); err != nil { + return fmt.Errorf("concurrent download aborted: %w", err) + } + + completedCount := completed.Load() + failedCount := failed.Load() + skippedCount := skipped.Load() + + logger.Info("Finished downloading copyright files", + slog.Int64("completed", completedCount), + slog.Int64("failed", failedCount), + slog.Int64("skipped", skippedCount), + slog.Int("total", total), + slog.Duration("elapsed", time.Since(startTime)), + ) + + if total > 0 { + failRatio := float64(failedCount) / float64(total) + if failRatio > maxFailureRate || (completedCount == 0 && total > 0) { + return fmt.Errorf("too many downloads failed: %d/%d (%.2f%% failed, max threshold is %.2f%%)", + failedCount, total, failRatio*100.0, maxFailureRate*100.0) + } + } + + return nil +} + +func logProgress(count int64, total int, startTime time.Time) { + if count%5000 == 0 || count == int64(total) { + pct := float64(count) / float64(total) * 100.0 + logger.Info("Download progress", + slog.Int64("completed", count), + slog.Int("total", total), + slog.Float64("percent", pct), + slog.Duration("elapsed", time.Since(startTime)), + ) + } +} + +// GenerateCurlConfiguration generates a curl config file for parallel downloads. +func GenerateCurlConfiguration(filelist []string, urlBase, configPath string) error { + file, err := os.Create(configPath) + if err != nil { + return fmt.Errorf("failed to create curl config file: %w", err) + } + defer file.Close() + + w := bufio.NewWriter(file) + for _, path := range filelist { + fullURL := fmt.Sprintf("%s/%s", strings.TrimRight(urlBase, "/"), strings.TrimLeft(path, "/")) + if _, err := fmt.Fprintf(w, "--output %s\nurl = %s\n", path, fullURL); err != nil { + return fmt.Errorf("failed to write curl config entry: %w", err) + } + } + + return w.Flush() +} + +// ExecuteCurl runs curl with the specified configuration file in the working directory. +func ExecuteCurl(ctx context.Context, configPath, workDir string) error { + if err := os.MkdirAll(workDir, 0755); err != nil { + return fmt.Errorf("failed to create work directory: %w", err) + } + + cmd := exec.CommandContext(ctx, "curl", "--parallel", "--create-dirs", "--config", configPath) + cmd.Dir = workDir + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + logger.Info("Executing curl in parallel", slog.String("config", configPath), slog.String("workDir", workDir)) + if err := cmd.Run(); err != nil { + return fmt.Errorf("curl execution failed: %w", err) + } + + return nil +} + +// UploadTarToGCS creates a tar archive of workDir and streams it directly to GCS. +func UploadTarToGCS(ctx context.Context, storageClient *storage.Client, workDir, gcsURI string) error { + bucketName, objectName, err := gcs.ParseGCSPath(gcsURI) + if err != nil { + return err + } + + logger.Info("Streaming tar archive to GCS", slog.String("bucket", bucketName), slog.String("object", objectName), slog.String("workDir", workDir)) + bkt := storageClient.Bucket(bucketName) + + pipeReader, pipeWriter := io.Pipe() + errChan := make(chan error, 1) + + go func() { + err := utility.CreateTarArchive(workDir, pipeWriter) + _ = pipeWriter.CloseWithError(err) + errChan <- err + }() + + uploadErr := gcs.UploadToGCS(ctx, bkt, objectName, pipeReader, "application/x-tar", nil) + archiveErr := <-errChan + + if uploadErr != nil { + return fmt.Errorf("failed to upload tar to GCS %s: %w", gcsURI, uploadErr) + } + if archiveErr != nil { + return fmt.Errorf("failed to create tar archive: %w", archiveErr) + } + + logger.Info("Successfully uploaded tar archive to GCS", slog.String("gcsURI", gcsURI)) + return nil +} + +func main() { + logger.InitGlobalLogger() + defer logger.Close() + + workDirFlag := flag.String("work-dir", "", "Directory to download copyright files into.") + filelistURL := flag.String("filelist-url", DefaultFilelistURL, "URL of the Debian filelist.yaml.xz file.") + urlBase := flag.String("url-base", DefaultURLBase, "Base URL for downloading Debian changelog/copyright files.") + prefixFilter := flag.String("prefix-filter", DefaultPrefixFilter, "Prefix filter for package paths to download (e.g. 'main/').") + minExpectedFiles := flag.Int("min-expected-files", DefaultMinExpectedFiles, "Minimum expected number of copyright files.") + numWorkers := flag.Int("workers", DefaultNumWorkers, "Number of concurrent download workers.") + skipExisting := flag.Bool("skip-existing", false, "Skip downloading files that already exist on disk with non-zero size.") + maxFailureRate := flag.Float64("max-failure-rate", DefaultMaxFailureRate, "Maximum allowable fraction of failed downloads before failing the job.") + gcsPath := flag.String("gcs-path", "", "Destination GCS path for tarball archive (e.g. gs://bucket/path/debian_copyright.tar). Defaults to GCS_PATH env var if unset.") + tarPath := flag.String("tar-path", "", "Optional local destination path for tarball archive (e.g. /scratch/debian_copyright.tar).") + useCurl := flag.Bool("use-curl", false, "If true, delegate downloads to curl --parallel instead of Go HTTP workers.") + curlConfigFile := flag.String("curl-config-file", "", "Optional path to write the generated curl configuration to.") + curlConfigOnly := flag.Bool("curl-config-only", false, "If true, only write the curl configuration file and exit.") + + flag.Parse() + + workDir := *workDirFlag + if workDir == "" && flag.NArg() > 0 { + workDir = flag.Arg(0) + } + if workDir == "" { + workDir = "." + } + + ctx := context.Background() + + poolSize := *numWorkers * 2 + if poolSize < 100 { + poolSize = 100 + } + + httpClient := &http.Client{ + Transport: &http.Transport{ + MaxIdleConns: poolSize, + MaxIdleConnsPerHost: poolSize, + IdleConnTimeout: 90 * time.Second, + ForceAttemptHTTP2: true, + }, + } + + logger.Info("Streaming and decompressing filelist", slog.String("url", *filelistURL)) + decompressedReader, err := utility.FetchAndDecompressXZ(ctx, httpClient, *filelistURL) + if err != nil { + logger.Fatal("Failed to fetch and decompress filelist", slog.Any("err", err)) + } + defer decompressedReader.Close() + + logger.Info("Extracting unstable copyright file paths", slog.String("prefixFilter", *prefixFilter)) + copyrightFiles, err := ExtractUnstableCopyright(decompressedReader, *prefixFilter) + if err != nil { + logger.Fatal("Failed to extract unstable copyright paths", slog.Any("err", err)) + } + _ = decompressedReader.Close() + + logger.Info("Discovered copyright files", slog.Int("count", len(copyrightFiles))) + + if len(copyrightFiles) < *minExpectedFiles { + logger.Fatal("Unexpectedly small number of copyright files found", + slog.Int("found", len(copyrightFiles)), + slog.Int("min_expected", *minExpectedFiles), + ) + } + + if *useCurl || *curlConfigOnly || *curlConfigFile != "" { + cfgPath := *curlConfigFile + if cfgPath == "" { + tempDir, err := os.MkdirTemp("", "debian-copyright-mirror-curl-*") + if err != nil { + logger.Fatal("Failed to create temporary directory for curl config", slog.Any("err", err)) + } + defer os.RemoveAll(tempDir) + cfgPath = filepath.Join(tempDir, "curl_configuration") + } + + logger.Info("Generating curl configuration", slog.String("path", cfgPath)) + if err := GenerateCurlConfiguration(copyrightFiles, *urlBase, cfgPath); err != nil { + logger.Fatal("Failed to generate curl configuration", slog.Any("err", err)) + } + + if *curlConfigOnly { + logger.Info("Curl configuration generated successfully; exiting as requested.") + return + } + + if *useCurl { + if err := ExecuteCurl(ctx, cfgPath, workDir); err != nil { + logger.Fatal("Curl download failed", slog.Any("err", err)) + } + } + } else { + if err := DownloadFilesConcurrently(ctx, httpClient, copyrightFiles, *urlBase, workDir, *numWorkers, *skipExisting, *maxFailureRate); err != nil { + logger.Fatal("Concurrent download failed", slog.Any("err", err)) + } + } + + if *tarPath != "" { + logger.Info("Creating local tar archive", slog.String("path", *tarPath), slog.String("workDir", workDir)) + if err := os.MkdirAll(filepath.Dir(*tarPath), 0755); err != nil { + logger.Fatal("Failed to create tar destination directory", slog.Any("err", err)) + } + tarFile, err := os.Create(*tarPath) + if err != nil { + logger.Fatal("Failed to create tar file", slog.Any("err", err)) + } + if err := utility.CreateTarArchive(workDir, tarFile); err != nil { + _ = tarFile.Close() + logger.Fatal("Failed to write tar archive", slog.Any("err", err)) + } + if err := tarFile.Close(); err != nil { + logger.Fatal("Failed to close tar file", slog.Any("err", err)) + } + logger.Info("Successfully created local tar archive", slog.String("path", *tarPath)) + } + + gcsDest := *gcsPath + if gcsDest == "" { + gcsDest = os.Getenv("GCS_PATH") + } + if gcsDest != "" { + storageClient, err := storage.NewClient(ctx) + if err != nil { + logger.Fatal("Failed to create GCS client", slog.Any("err", err)) + } + defer storageClient.Close() + + if err := UploadTarToGCS(ctx, storageClient, workDir, gcsDest); err != nil { + logger.Fatal("Failed to upload tar archive to GCS", slog.Any("err", err)) + } + } + + logger.Info("Debian copyright mirror sync completed successfully.", slog.String("workDir", workDir)) +} diff --git a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main_test.go b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main_test.go new file mode 100644 index 00000000000..2d1b39a974a --- /dev/null +++ b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main_test.go @@ -0,0 +1,193 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "reflect" + "strings" + "testing" +) + +const sampleFilelistYAML = ` +0ad: + 0.0.17-1: + - main/0/0ad/0ad_0.0.17-1_copyright + - main/0/0ad/0ad_0.0.17-1_changelog + testing: + - main/0/0ad/testing_copyright + - main/0/0ad/testing_changelog + unstable: + - main/0/0ad/unstable_changelog + - main/0/0ad/unstable_copyright + - main/0/0ad/unstable_NEWS +0ad-data: + unstable: + - main/0/0ad-data/unstable_changelog + - main/0/0ad-data/unstable_copyright +non-free-pkg: + unstable: + - non-free/n/non-free-pkg/unstable_changelog + - non-free/n/non-free-pkg/unstable_copyright +no-unstable-pkg: + stable: + - main/n/no-unstable-pkg/stable_copyright +no-copyright-pkg: + unstable: + - main/n/no-copyright-pkg/unstable_changelog +'0xffff': + 0.6.1-1: + - main/0/0xffff/0xffff_0.6.1-1_changelog + unstable: + - "main/0/0xffff/unstable_changelog" + - 'main/0/0xffff/unstable_copyright' +` + +func TestExtractUnstableCopyright(t *testing.T) { + tests := []struct { + name string + yamlContent string + prefixFilter string + expected []string + }{ + { + name: "Sample filelist with main/ filter", + yamlContent: sampleFilelistYAML, + prefixFilter: "main/", + expected: []string{ + "main/0/0ad/unstable_copyright", + "main/0/0ad-data/unstable_copyright", + "main/0/0xffff/unstable_copyright", + }, + }, + { + name: "Sample filelist without filter", + yamlContent: sampleFilelistYAML, + prefixFilter: "", + expected: []string{ + "main/0/0ad/unstable_copyright", + "main/0/0ad-data/unstable_copyright", + "non-free/n/non-free-pkg/unstable_copyright", + "main/0/0xffff/unstable_copyright", + }, + }, + { + name: "Empty YAML", + yamlContent: "", + prefixFilter: "main/", + expected: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := strings.NewReader(tt.yamlContent) + got, err := ExtractUnstableCopyright(r, tt.prefixFilter) + if err != nil { + t.Fatalf("ExtractUnstableCopyright unexpected error: %v", err) + } + if !reflect.DeepEqual(got, tt.expected) { + t.Errorf("ExtractUnstableCopyright() = %v, want %v", got, tt.expected) + } + }) + } +} + +func TestGenerateCurlConfiguration(t *testing.T) { + tempDir := t.TempDir() + configPath := filepath.Join(tempDir, "curl_config") + + files := []string{ + "main/0/0ad/unstable_copyright", + "main/0/0xffff/unstable_copyright", + } + urlBase := "https://metadata.ftp-master.debian.org/changelogs" + + if err := GenerateCurlConfiguration(files, urlBase, configPath); err != nil { + t.Fatalf("GenerateCurlConfiguration returned error: %v", err) + } + + content, err := os.ReadFile(configPath) + if err != nil { + t.Fatalf("Failed to read generated config: %v", err) + } + + expected := "--output main/0/0ad/unstable_copyright\n" + + "url = https://metadata.ftp-master.debian.org/changelogs/main/0/0ad/unstable_copyright\n" + + "--output main/0/0xffff/unstable_copyright\n" + + "url = https://metadata.ftp-master.debian.org/changelogs/main/0/0xffff/unstable_copyright\n" + + if string(content) != expected { + t.Errorf("GenerateCurlConfiguration() =\n%s\nwant:\n%s", string(content), expected) + } +} + +func TestDownloadFilesConcurrently(t *testing.T) { + fileMap := map[string]string{ + "main/a/pkg1/unstable_copyright": "Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/\nSource: https://github.com/foo/pkg1\n", + "main/b/pkg2/unstable_copyright": "Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/\nSource: https://github.com/bar/pkg2\n", + } + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + path := strings.TrimPrefix(r.URL.Path, "/") + if content, ok := fileMap[path]; ok { + w.WriteHeader(http.StatusOK) + _, _ = fmt.Fprint(w, content) + return + } + w.WriteHeader(http.StatusNotFound) + })) + defer ts.Close() + + tempDir := t.TempDir() + ctx := context.Background() + client := ts.Client() + + filelist := []string{ + "main/a/pkg1/unstable_copyright", + "main/b/pkg2/unstable_copyright", + } + + if err := DownloadFilesConcurrently(ctx, client, filelist, ts.URL, tempDir, 2, false, 0.05); err != nil { + t.Fatalf("DownloadFilesConcurrently failed: %v", err) + } + + for _, relPath := range filelist { + fullPath := filepath.Join(tempDir, relPath) + data, err := os.ReadFile(fullPath) + if err != nil { + t.Errorf("Failed to read %s: %v", relPath, err) + continue + } + if string(data) != fileMap[relPath] { + t.Errorf("Content mismatch for %s: got %q, want %q", relPath, string(data), fileMap[relPath]) + } + } + + // Test failure threshold breach + badFileList := []string{ + "main/missing/1", + "main/missing/2", + } + err := DownloadFilesConcurrently(ctx, client, badFileList, ts.URL, tempDir, 2, false, 0.05) + if err == nil { + t.Errorf("Expected failure threshold error, got nil") + } +} diff --git a/vulnfeeds/gcs-tools/gcs.go b/vulnfeeds/gcs-tools/gcs.go index 8fc78d7b4ff..deb217c17e3 100644 --- a/vulnfeeds/gcs-tools/gcs.go +++ b/vulnfeeds/gcs-tools/gcs.go @@ -145,6 +145,19 @@ func UploadToGCS(ctx context.Context, bkt *storage.BucketHandle, objectName stri return nil } +// ParseGCSPath parses a gs:// URI into bucket and object name components. +func ParseGCSPath(gcsURI string) (bucket, object string, err error) { + if !strings.HasPrefix(gcsURI, "gs://") { + return "", "", fmt.Errorf("invalid GCS URI: %s (must start with gs://)", gcsURI) + } + trimmed := strings.TrimPrefix(gcsURI, "gs://") + parts := strings.SplitN(trimmed, "/", 2) + if len(parts) < 2 || parts[0] == "" || parts[1] == "" { + return "", "", fmt.Errorf("invalid GCS URI format %q (expected gs://bucket/object)", gcsURI) + } + return parts[0], parts[1], nil +} + // UploadFile uploads a local file to a GCS bucket. func UploadFile(ctx context.Context, bkt *storage.BucketHandle, objectName string, filePath string) error { f, err := os.Open(filePath) diff --git a/vulnfeeds/gcs-tools/gcs_test.go b/vulnfeeds/gcs-tools/gcs_test.go index 65d0514ad77..01cdf5deb61 100644 --- a/vulnfeeds/gcs-tools/gcs_test.go +++ b/vulnfeeds/gcs-tools/gcs_test.go @@ -299,3 +299,50 @@ func TestListObjectsFast(t *testing.T) { t.Errorf("ListObjectsFast returned unexpected list.\ngot: %v\nwant: %v", got, expected) } } + +func TestParseGCSPath(t *testing.T) { + tests := []struct { + input string + wantBucket string + wantObject string + expectError bool + }{ + { + input: "gs://my-bucket/path/to/file.tar", + wantBucket: "my-bucket", + wantObject: "path/to/file.tar", + expectError: false, + }, + { + input: "gs://cve-osv-conversion/debian_copyright.tar", + wantBucket: "cve-osv-conversion", + wantObject: "debian_copyright.tar", + expectError: false, + }, + { + input: "/local/path/file.tar", + expectError: true, + }, + { + input: "gs://bucket-only", + expectError: true, + }, + { + input: "gs://", + expectError: true, + }, + } + + for _, tt := range tests { + bucket, obj, err := ParseGCSPath(tt.input) + if (err != nil) != tt.expectError { + t.Errorf("ParseGCSPath(%q) error = %v, expectError = %v", tt.input, err, tt.expectError) + continue + } + if !tt.expectError { + if bucket != tt.wantBucket || obj != tt.wantObject { + t.Errorf("ParseGCSPath(%q) = (%q, %q), want (%q, %q)", tt.input, bucket, obj, tt.wantBucket, tt.wantObject) + } + } + } +} diff --git a/vulnfeeds/utility/archive.go b/vulnfeeds/utility/archive.go new file mode 100644 index 00000000000..267b8e4ba4e --- /dev/null +++ b/vulnfeeds/utility/archive.go @@ -0,0 +1,118 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utility + +import ( + "archive/tar" + "fmt" + "io" + "os" + "path/filepath" + "strings" +) + +// CreateTarArchive archives all files and subdirectories in sourceDir into the tar writer. +func CreateTarArchive(sourceDir string, w io.Writer) error { + tw := tar.NewWriter(w) + defer tw.Close() + + sourceDir = filepath.Clean(sourceDir) + return filepath.Walk(sourceDir, func(file string, fi os.FileInfo, err error) error { + if err != nil { + return err + } + if file == sourceDir { + return nil + } + + relPath, err := filepath.Rel(sourceDir, file) + if err != nil { + return fmt.Errorf("failed to get relative path for %s: %w", file, err) + } + + tarName := filepath.ToSlash(relPath) + if fi.IsDir() { + tarName += "/" + } + + hdr, err := tar.FileInfoHeader(fi, "") + if err != nil { + return fmt.Errorf("failed to create tar header for %s: %w", file, err) + } + hdr.Name = tarName + + if err := tw.WriteHeader(hdr); err != nil { + return fmt.Errorf("failed to write tar header for %s: %w", file, err) + } + + if fi.Mode().IsRegular() { + f, err := os.Open(file) + if err != nil { + return fmt.Errorf("failed to open file %s: %w", file, err) + } + defer f.Close() + + if _, err := io.Copy(tw, f); err != nil { + return fmt.Errorf("failed to copy file %s to tar: %w", file, err) + } + } + + return nil + }) +} + +// ExtractTarArchive extracts a tar archive stream into the destination directory. +func ExtractTarArchive(r io.Reader, destDir string) error { + tr := tar.NewReader(r) + cleanDest := filepath.Clean(destDir) + + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return fmt.Errorf("error reading tar entry: %w", err) + } + + // Prevent Zip Slip / path traversal + targetPath := filepath.Join(cleanDest, filepath.FromSlash(hdr.Name)) + if !strings.HasPrefix(targetPath, cleanDest+string(os.PathSeparator)) && targetPath != cleanDest { + return fmt.Errorf("illegal file path in tar archive: %s", hdr.Name) + } + + switch hdr.Typeflag { + case tar.TypeDir: + if err := os.MkdirAll(targetPath, 0755); err != nil { + return fmt.Errorf("failed to create directory %s: %w", targetPath, err) + } + case tar.TypeReg: + if err := os.MkdirAll(filepath.Dir(targetPath), 0755); err != nil { + return fmt.Errorf("failed to create directory for file %s: %w", targetPath, err) + } + outFile, err := os.OpenFile(targetPath, os.O_CREATE|os.O_RDWR|os.O_TRUNC, hdr.FileInfo().Mode()) + if err != nil { + return fmt.Errorf("failed to create file %s: %w", targetPath, err) + } + if _, err := io.Copy(outFile, tr); err != nil { + outFile.Close() + return fmt.Errorf("failed to write file content %s: %w", targetPath, err) + } + outFile.Close() + } + } + + return nil +} diff --git a/vulnfeeds/utility/archive_test.go b/vulnfeeds/utility/archive_test.go new file mode 100644 index 00000000000..a853f771bf1 --- /dev/null +++ b/vulnfeeds/utility/archive_test.go @@ -0,0 +1,69 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utility + +import ( + "bytes" + "os" + "path/filepath" + "testing" +) + +func TestTarArchiveRoundtrip(t *testing.T) { + tempDir := t.TempDir() + sourceDir := filepath.Join(tempDir, "source") + destDir := filepath.Join(tempDir, "dest") + + if err := os.MkdirAll(filepath.Join(sourceDir, "sub"), 0755); err != nil { + t.Fatalf("Failed to create test source dir: %v", err) + } + + file1 := filepath.Join(sourceDir, "file1.txt") + file2 := filepath.Join(sourceDir, "sub", "file2.txt") + if err := os.WriteFile(file1, []byte("hello archive"), 0644); err != nil { + t.Fatalf("Failed to write file1: %v", err) + } + if err := os.WriteFile(file2, []byte("nested file content"), 0644); err != nil { + t.Fatalf("Failed to write file2: %v", err) + } + + var buf bytes.Buffer + if err := CreateTarArchive(sourceDir, &buf); err != nil { + t.Fatalf("CreateTarArchive failed: %v", err) + } + + if err := ExtractTarArchive(&buf, destDir); err != nil { + t.Fatalf("ExtractTarArchive failed: %v", err) + } + + destFile1 := filepath.Join(destDir, "file1.txt") + destFile2 := filepath.Join(destDir, "sub", "file2.txt") + + data1, err := os.ReadFile(destFile1) + if err != nil { + t.Fatalf("Failed to read extracted file1: %v", err) + } + if string(data1) != "hello archive" { + t.Errorf("file1 content = %q, want %q", string(data1), "hello archive") + } + + data2, err := os.ReadFile(destFile2) + if err != nil { + t.Fatalf("Failed to read extracted file2: %v", err) + } + if string(data2) != "nested file content" { + t.Errorf("file2 content = %q, want %q", string(data2), "nested file content") + } +} diff --git a/vulnfeeds/utility/decompress.go b/vulnfeeds/utility/decompress.go new file mode 100644 index 00000000000..8f80ea2d0f3 --- /dev/null +++ b/vulnfeeds/utility/decompress.go @@ -0,0 +1,127 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utility + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "os" + "os/exec" +) + +// SubprocessReadCloser wraps a command's stdout pipe and ensures the subprocess +// is waited for and input resources are closed when Close() is called. +type SubprocessReadCloser struct { + io.ReadCloser + cmd *exec.Cmd + bodyToClose io.Closer +} + +// Close closes the underlying reader pipe, any input reader, and waits for the subprocess. +func (s *SubprocessReadCloser) Close() error { + var errs []error + if s.ReadCloser != nil { + if err := s.ReadCloser.Close(); err != nil { + errs = append(errs, err) + } + } + if s.bodyToClose != nil { + if err := s.bodyToClose.Close(); err != nil { + errs = append(errs, err) + } + } + if s.cmd != nil { + if err := s.cmd.Wait(); err != nil { + var exitErr *exec.ExitError + if errors.As(err, &exitErr) && !exitErr.Success() { + errs = append(errs, err) + } + } + } + return errors.Join(errs...) +} + +func getXZDecompressCommand() (string, error) { + if _, err := exec.LookPath("xz"); err == nil { + return "xz", nil + } + if _, err := exec.LookPath("unxz"); err == nil { + return "unxz", nil + } + return "", errors.New("neither 'xz' nor 'unxz' command-line tool found in PATH") +} + +// DecompressXZ streams an xz-compressed reader through the system xz or unxz command. +func DecompressXZ(ctx context.Context, r io.Reader) (io.ReadCloser, error) { + decompressCmd, err := getXZDecompressCommand() + if err != nil { + return nil, err + } + + cmd := exec.CommandContext(ctx, decompressCmd, "-dc") + cmd.Stdin = r + stdout, err := cmd.StdoutPipe() + if err != nil { + return nil, fmt.Errorf("failed to create stdout pipe for %s: %w", decompressCmd, err) + } + + if err := cmd.Start(); err != nil { + return nil, fmt.Errorf("failed to start %s: %w", decompressCmd, err) + } + + var bodyToClose io.Closer + if closer, ok := r.(io.Closer); ok { + bodyToClose = closer + } + + return &SubprocessReadCloser{ + ReadCloser: stdout, + cmd: cmd, + bodyToClose: bodyToClose, + }, nil +} + +// DecompressXZFile decompresses a local xz-compressed file. +func DecompressXZFile(ctx context.Context, xzFilePath string) (io.ReadCloser, error) { + f, err := os.Open(xzFilePath) + if err != nil { + return nil, fmt.Errorf("failed to open %s: %w", xzFilePath, err) + } + + return DecompressXZ(ctx, f) +} + +// FetchAndDecompressXZ fetches an xz-compressed URL over HTTP and streams +// its decompressed content directly through xz/unxz without saving to disk. +func FetchAndDecompressXZ(ctx context.Context, client *http.Client, url string) (io.ReadCloser, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, fmt.Errorf("failed to create request for %s: %w", url, err) + } + + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to fetch %s: %w", url, err) + } + if resp.StatusCode != http.StatusOK { + resp.Body.Close() + return nil, fmt.Errorf("HTTP status %d fetching %s", resp.StatusCode, url) + } + + return DecompressXZ(ctx, resp.Body) +} diff --git a/vulnfeeds/utility/decompress_test.go b/vulnfeeds/utility/decompress_test.go new file mode 100644 index 00000000000..fe404b02da1 --- /dev/null +++ b/vulnfeeds/utility/decompress_test.go @@ -0,0 +1,108 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utility + +import ( + "bytes" + "context" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" +) + +var testXZPayload = []byte{ + 0xfd, 0x37, 0x7a, 0x58, 0x5a, 0x00, 0x00, 0x04, 0xe6, 0xd6, 0xb4, 0x46, + 0x02, 0x00, 0x21, 0x01, 0x16, 0x00, 0x00, 0x00, 0x74, 0x2f, 0xe5, 0xa3, + 0x01, 0x00, 0x1d, 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x44, 0x65, 0x62, + 0x69, 0x61, 0x6e, 0x20, 0x43, 0x6f, 0x70, 0x79, 0x72, 0x69, 0x67, 0x68, + 0x74, 0x20, 0x4d, 0x69, 0x72, 0x72, 0x6f, 0x72, 0x21, 0x00, 0x00, 0x00, + 0xc7, 0x2f, 0x01, 0x8a, 0x65, 0x5a, 0x9a, 0x97, 0x00, 0x01, 0x36, 0x1e, + 0x3d, 0x19, 0x95, 0x53, 0x1f, 0xb6, 0xf3, 0x7d, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x04, 0x59, 0x5a, +} + +func TestDecompressXZ(t *testing.T) { + ctx := context.Background() + reader, err := DecompressXZ(ctx, bytes.NewReader(testXZPayload)) + if err != nil { + t.Fatalf("DecompressXZ failed: %v", err) + } + defer reader.Close() + + data, err := io.ReadAll(reader) + if err != nil { + t.Fatalf("Failed to read decompressed data: %v", err) + } + + want := "Hello Debian Copyright Mirror!" + if string(data) != want { + t.Errorf("Decompressed = %q, want %q", string(data), want) + } +} + +func TestDecompressXZFile(t *testing.T) { + tempDir := t.TempDir() + filePath := filepath.Join(tempDir, "test.xz") + if err := os.WriteFile(filePath, testXZPayload, 0644); err != nil { + t.Fatalf("Failed to write test file: %v", err) + } + + ctx := context.Background() + reader, err := DecompressXZFile(ctx, filePath) + if err != nil { + t.Fatalf("DecompressXZFile failed: %v", err) + } + defer reader.Close() + + data, err := io.ReadAll(reader) + if err != nil { + t.Fatalf("Failed to read decompressed data: %v", err) + } + + want := "Hello Debian Copyright Mirror!" + if string(data) != want { + t.Errorf("Decompressed = %q, want %q", string(data), want) + } +} + +func TestFetchAndDecompressXZ(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write(testXZPayload) + })) + defer ts.Close() + + ctx := context.Background() + client := ts.Client() + + reader, err := FetchAndDecompressXZ(ctx, client, ts.URL+"/file.xz") + if err != nil { + t.Fatalf("FetchAndDecompressXZ failed: %v", err) + } + defer reader.Close() + + data, err := io.ReadAll(reader) + if err != nil { + t.Fatalf("Failed to read decompressed data: %v", err) + } + + want := "Hello Debian Copyright Mirror!" + if string(data) != want { + t.Errorf("Decompressed = %q, want %q", string(data), want) + } +} diff --git a/vulnfeeds/utility/download.go b/vulnfeeds/utility/download.go new file mode 100644 index 00000000000..1b2bed79635 --- /dev/null +++ b/vulnfeeds/utility/download.go @@ -0,0 +1,102 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utility + +import ( + "context" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "time" + + "github.com/sethvargo/go-retry" +) + +const ( + DefaultDownloadTimeout = 30 * time.Second +) + +// DownloadFile downloads a URL to a local destination file with automatic retries, +// safe temporary file writing, and on-demand directory creation. +// If skipExisting is true and destPath exists with non-zero size, the download is skipped. +func DownloadFile(ctx context.Context, client *http.Client, url, destPath string, skipExisting bool) error { + if skipExisting { + if fi, err := os.Stat(destPath); err == nil && fi.Size() > 0 { + return nil + } + } + + backoff := retry.NewExponential(1 * time.Second) + backoff = retry.WithMaxRetries(3, backoff) + + return retry.Do(ctx, backoff, func(ctx context.Context) error { + reqCtx, cancel := context.WithTimeout(ctx, DefaultDownloadTimeout) + defer cancel() + + req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, url, nil) + if err != nil { + return fmt.Errorf("failed to create request for %s: %w", url, err) + } + + resp, err := client.Do(req) + if err != nil { + return retry.RetryableError(fmt.Errorf("HTTP request failed for %s: %w", url, err)) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + if resp.StatusCode >= 500 || resp.StatusCode == http.StatusTooManyRequests { + return retry.RetryableError(fmt.Errorf("HTTP status %d for %s", resp.StatusCode, url)) + } + return fmt.Errorf("HTTP status %d for %s", resp.StatusCode, url) + } + + dir := filepath.Dir(destPath) + tmpFile, err := os.CreateTemp(dir, "download-*") + if err != nil { + if os.IsNotExist(err) { + if mkdirErr := os.MkdirAll(dir, 0755); mkdirErr != nil { + return fmt.Errorf("failed to create directory %s: %w", dir, mkdirErr) + } + tmpFile, err = os.CreateTemp(dir, "download-*") + } + if err != nil { + return fmt.Errorf("failed to create temp file in %s: %w", dir, err) + } + } + tmpPath := tmpFile.Name() + + _, copyErr := io.Copy(tmpFile, resp.Body) + closeErr := tmpFile.Close() + + if copyErr != nil { + _ = os.Remove(tmpPath) + return retry.RetryableError(fmt.Errorf("failed to write data: %w", copyErr)) + } + if closeErr != nil { + _ = os.Remove(tmpPath) + return fmt.Errorf("failed to close temp file: %w", closeErr) + } + + if err := os.Rename(tmpPath, destPath); err != nil { + _ = os.Remove(tmpPath) + return fmt.Errorf("failed to move temp file to %s: %w", destPath, err) + } + + return nil + }) +} diff --git a/vulnfeeds/utility/download_test.go b/vulnfeeds/utility/download_test.go new file mode 100644 index 00000000000..d1eb62b48e4 --- /dev/null +++ b/vulnfeeds/utility/download_test.go @@ -0,0 +1,66 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utility + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" +) + +func TestDownloadFile(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/file.txt" { + w.WriteHeader(http.StatusOK) + _, _ = fmt.Fprint(w, "downloaded content") + return + } + w.WriteHeader(http.StatusNotFound) + })) + defer ts.Close() + + tempDir := t.TempDir() + destPath := filepath.Join(tempDir, "nested", "dir", "file.txt") + + ctx := context.Background() + client := ts.Client() + + if err := DownloadFile(ctx, client, ts.URL+"/file.txt", destPath, false); err != nil { + t.Fatalf("DownloadFile failed: %v", err) + } + + data, err := os.ReadFile(destPath) + if err != nil { + t.Fatalf("Failed to read downloaded file: %v", err) + } + if string(data) != "downloaded content" { + t.Errorf("content = %q, want %q", string(data), "downloaded content") + } + + // Test skipExisting = true + if err := DownloadFile(ctx, client, ts.URL+"/404-should-skip", destPath, true); err != nil { + t.Fatalf("DownloadFile with skipExisting should have succeeded without error, got %v", err) + } + + // Test 404 + err = DownloadFile(ctx, client, ts.URL+"/nonexistent", filepath.Join(tempDir, "404.txt"), false) + if err == nil { + t.Errorf("Expected error for 404, got nil") + } +} From 9d4392076bc452d7ccfcf5bd87ea2daf8bcdd634 Mon Sep 17 00:00:00 2001 From: Jess Lowe Date: Wed, 19 Aug 2026 00:27:55 +0000 Subject: [PATCH 2/7] some other files + readme details --- .../mirrors/debian-copyright-mirror/README.md | 121 +++++++++++++ .../mirrors/debian-copyright-mirror/build.sh | 4 +- .../debian-copyright-mirror.py | 163 ------------------ .../debian-copyright-mirror.sh | 17 +- 4 files changed, 129 insertions(+), 176 deletions(-) create mode 100644 vulnfeeds/cmd/mirrors/debian-copyright-mirror/README.md delete mode 100644 vulnfeeds/cmd/mirrors/debian-copyright-mirror/debian-copyright-mirror.py diff --git a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/README.md b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/README.md new file mode 100644 index 00000000000..eb139d03051 --- /dev/null +++ b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/README.md @@ -0,0 +1,121 @@ +# Debian Copyright Mirror + +`debian-copyright-mirror` maintains a mirror of machine-readable Debian package copyright files for packages in Debian `unstable` (`main` section). + +--- + +## Purpose & Usage in OSV + +In the OSV and vulnerability feeds pipeline, correlating Common Vulnerabilities and Exposures (CVE) entries and Common Platform Enumeration (CPE) identifiers with upstream open-source source code repositories (e.g., on GitHub, GitLab, or Git) is a crucial step for automated vulnerability management. + +### The Problem +- CVE entries and NVD CPE dictionaries often lack structured upstream repository URLs, or only provide arbitrary website references. +- Without a reliable mapping from package names and CPE products to source code repositories, automated tools cannot easily determine affected Git commits, tags, or version ranges for Git-based vulnerability matching. + +### How Debian Copyright Files Solve This +Debian packages in `unstable` follow the [Machine-readable debian/copyright format (DEP-5 / Copyright Specification 1.0)](https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/). These files contain structured metadata in header paragraphs, notably: + +```text +Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: example-lib +Source: https://github.com/example/example-lib +``` + +The `Source:` field provides a canonical, maintainer-verified URL to the upstream open-source repository. + +### Downstream Consumers +1. **`cpe-repo-gen`** (`cmd/mirrors/cpe-repo-gen`): + - Downloads the `debian_copyright.tar` archive generated by this tool from Google Cloud Storage (`gs://cve-osv-conversion/debian_copyright/debian_copyright.tar`). + - Scans the mirrored `unstable_copyright` files to resolve Debian package names and CPE products to upstream source code repositories. + - Produces the canonical `cpe_product_to_repo.json` mapping. +2. **OSV Conversion & Enrichment Pipelines**: + - Use the repository mappings derived from Debian copyright data to accurately calculate affected commit ranges, detect fixed revisions, and generate OSV-format records for Debian, NVD, and other feeds. + +--- + +## How It Works + +```mermaid +graph TD + A["Debian FTP Master
(filelist.yaml.xz)"] -->|HTTP Stream & xz -dc| B["debian-copyright-mirror
(Go binary)"] + B -->|Streaming YAML Parser| C["Extract ~44k+ main/
unstable_copyright paths"] + C -->|Channel-based Worker Pool| D["Download copyright files
to local directory"] + D -->|Validate failure rate & count| E["Native Tar Streaming"] + E -->|GCS Client Writer| F["GCS Bucket
(gs://cve-osv-conversion/...)"] + F -->|tar -xf| G["cpe-repo-gen
(CPE to Repo Mapping)"] +``` + +1. **Manifest Streaming & Decompression**: Fetches `filelist.yaml.xz` from `https://metadata.ftp-master.debian.org/changelogs/filelist.yaml.xz` over HTTP and streams it directly through `xz -dc` stdin without intermediate disk roundtrips. +2. **Manifest Parsing**: Parses the YAML manifest to discover all packages containing an `unstable_copyright` entry under the `unstable` suite in the `main` archive section. +3. **Validation**: Asserts that the number of discovered files meets a sanity threshold (by default, at least 40,000 files) to ensure upstream feeds were not corrupted or truncated. +4. **Optimized Concurrent Downloads**: + - Spawns a channel-based worker pool (default 50 workers) with HTTP/2 keep-alive connection reuse and exponential backoff retries. + - Optimized file and directory creation (avoids redundant `MkdirAll` calls on existing directories). + - Tracks success/failure metrics and enforces a maximum failure threshold (`-max-failure-rate`) to prevent false-positive completions on network or CDN outages. + - Supports incremental runs with `-skip-existing` to skip re-downloading unchanged files. + - Optional: Delegated download via `curl --parallel` using generated curl configuration files (`-use-curl`). +5. **Native Archival & GCS Streaming**: Packages the downloaded mirror into a `.tar` archive and streams it directly to Google Cloud Storage (e.g. `gs://cve-osv-conversion/debian_copyright/debian_copyright.tar`) using the Cloud Storage client. + +--- + +## CLI Usage + +### Running Locally + +```bash +# Build and run directly using Go +go run ./cmd/mirrors/debian-copyright-mirror [flags] [work_dir] +``` + +Examples: + +```bash +# Download copyright files into /tmp/debian_copyright using 50 workers +go run ./cmd/mirrors/debian-copyright-mirror -workers 50 /tmp/debian_copyright + +# Incremental run skipping already downloaded files +go run ./cmd/mirrors/debian-copyright-mirror -skip-existing /tmp/debian_copyright + +# Download and create a local tar archive +go run ./cmd/mirrors/debian-copyright-mirror -tar-path /scratch/debian_copyright.tar /tmp/debian_copyright + +# Download and stream directly to GCS +go run ./cmd/mirrors/debian-copyright-mirror -gcs-path gs://my-bucket/debian_copyright.tar /tmp/debian_copyright +``` + +### Flags + +| Flag | Type | Default | Description | +| :--- | :--- | :--- | :--- | +| `-work-dir` | string | `.` | Target directory to save downloaded copyright files (can also be passed as the first positional argument). | +| `-workers` | int | `50` | Number of concurrent download workers. | +| `-skip-existing` | bool | `false` | Skip downloading files that already exist on disk with non-zero size. | +| `-max-failure-rate` | float | `0.05` | Maximum allowable fraction of failed downloads (e.g. `0.05` = 5%) before failing the job. | +| `-gcs-path` | string | `""` | Destination GCS path for the tarball archive (e.g. `gs://bucket/debian_copyright.tar`). Defaults to `$GCS_PATH` if unset. | +| `-tar-path` | string | `""` | Optional local destination path for tarball archive. | +| `-filelist-url` | string | `https://metadata.ftp-master.debian.org/changelogs/filelist.yaml.xz` | URL of the Debian filelist YAML archive. | +| `-url-base` | string | `https://metadata.ftp-master.debian.org/changelogs` | Base URL for downloading individual copyright files. | +| `-prefix-filter` | string | `main/` | Archive section prefix to filter (e.g., `main/`). | +| `-min-expected-files` | int | `40000` | Minimum expected number of copyright files; fails if fewer are found. | +| `-use-curl` | bool | `false` | Delegate downloads to `curl --parallel` instead of Go worker pool. | +| `-curl-config-file` | string | `""` | Optional path to output the generated curl configuration file. | +| `-curl-config-only` | bool | `false` | Generate the curl configuration file and exit immediately without downloading. | + +--- + +## Docker & Deployment + +### Building the Docker Container + +Build the container image from the `vulnfeeds/` directory: + +```bash +docker build -t gcr.io/oss-vdb/debian-copyright-mirror:latest -f cmd/mirrors/debian-copyright-mirror/Dockerfile . +``` + +### Kubernetes CronJob Deployment + +In production, this mirror runs as a scheduled Kubernetes `CronJob` in GKE (defined in `deployment/clouddeploy/gke-workers/base/feeds/debian-copyright-mirror.yaml` and environment overlays): + +- **Schedule**: Runs daily (`0 5 * * *` Sydney time). +- **Entrypoint**: Executes `debian-copyright-mirror.sh`, which runs `debian-copyright-mirror` with `-gcs-path "${GCS_PATH}"` to stream the archive directly to `gs://cve-osv-conversion/debian_copyright/debian_copyright.tar`. diff --git a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/build.sh b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/build.sh index 1fae76fdb88..375eb4cdaf3 100755 --- a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/build.sh +++ b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/build.sh @@ -16,7 +16,9 @@ set -ex # See the License for the specific language governing permissions and # limitations under the License. +cd ../../.. + docker build \ -t gcr.io/oss-vdb/debian-copyright-mirror:latest \ - -f Dockerfile --pull . && \ + -f cmd/mirrors/debian-copyright-mirror/Dockerfile --pull . && \ gcloud docker -- push gcr.io/oss-vdb/debian-copyright-mirror:latest diff --git a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/debian-copyright-mirror.py b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/debian-copyright-mirror.py deleted file mode 100644 index c4bdfe2d202..00000000000 --- a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/debian-copyright-mirror.py +++ /dev/null @@ -1,163 +0,0 @@ -""" -Download all of the copyright files for packages in main in Debian unstable. - -This: - -Parses https://metadata.ftp-master.debian.org/changelogs/filelist.yaml.xz - to identify the files to retrieve -Generates a curl configuration to download the URLs -Executes curl to download the URLs in parallel -""" - -import os -import argparse -import yaml -import lzma -import urllib.request -import subprocess -from typing import List, Optional, Dict - - -class Error(Exception): - """General purpose error class.""" - - -def download_url_to_directory(url: str, - directory: str, - filename: Optional[str] = None) -> Optional[str]: - """ - Downloads a URL to a specified directory. - - Args: - url (str): The URL to download. - directory (str): The directory to save the file to. - filename (str, optional): The filename to use. If None, the filename - is extracted from the URL. Defaults to None. - - Returns: - str: The full path to the downloaded file, or None on error. - """ - try: - if not os.path.exists(directory): - os.makedirs(directory) - - if filename is None: - filename = os.path.basename(urllib.parse.urlsplit(url).path) - - filepath = os.path.join(directory, filename) - - urllib.request.urlretrieve(url, filepath) - return filepath - - except urllib.error.URLError as e: - print(f'Error downloading {url}: {e}') - return None - except OSError as e: # Catch directory creation/file writing errors. - print(f'OS Error: {e}') - return None - except Exception as e: # Catch any other unexpected error. - print(f'An unexpected error occurred: {e}') - return None - - -def extract_unstable_copyright(filelist: str) -> Dict: - """ - Extracts the 'unstable_copyright' entry for each package - from an xz-compressed YAML file. - - Args: - filelist (str): The path to the xz-compressed YAML filelist - - Returns: - A dictionary where keys are package names and values are - their 'unstable_copyright' entries, or None if no unstable - copyright is found. - """ - try: - with lzma.open(filelist, 'rt', encoding='utf-8') as f: - data = yaml.safe_load(f) - - results = {} - for package, versions in data.items(): - if 'unstable' in versions: - entries = versions['unstable'] - for entry in entries: - if entry.endswith('unstable_copyright'): - results[package] = entry - break # Found it, no need to continue checking this package. - - return results - - except FileNotFoundError: - print(f"Error: File not found at {filelist}") - return None - except lzma.LZMAError as e: - print(f"Error: LZMA decompression failed: {e}") - return None - except yaml.YAMLError as e: - print(f"Error: YAML parsing failed: {e}") - return None - except Exception as e: - print(f"An unexpected error occurred: {e}") - return None - - -def generate_curl_configuration(filelist: List[str]): - """ - Generates a curl configuration to download all of the files in filelist. - - --output filename - url = https://url - - Args: - filelist (List[str]): a list of files to download. - """ - - url_base = 'https://metadata.ftp-master.debian.org/changelogs' - - with open('/tmp/curl_configuration', 'w') as curl_config: - curl_config.writelines([ - '--output ' + path + '\n' + 'url = ' + os.path.join(url_base, path) + - '\n' for path in filelist - ]) - - -def execute_curl(configuration: str, directory: str): - """ - Execute curl with the supplied configuration in the specified directory. - - Args: - configuration (str): path to configuration file. - directory (str): path to set current working directory to. - """ - - os.makedirs(directory) - subprocess.run( - ['curl', '--parallel', '--create-dirs', '--config', configuration], - cwd=directory, - check=True) - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument('work_dir') - args = parser.parse_args() - - download_url_to_directory( - 'https://metadata.ftp-master.debian.org/changelogs/filelist.yaml.xz', - '/tmp') - unstable_package_copyright_files = extract_unstable_copyright( - '/tmp/filelist.yaml.xz') - if unstable_package_copyright_files is None: - raise Error('Unexpected result determining files to download') - generate_curl_configuration( - f for f in unstable_package_copyright_files.values() - if f.startswith('main/')) - with open("/tmp/curl_configuration") as curl_configuration: - if len(curl_configuration.readlines()) < 80000: - raise Error('Unexpectly small curl configuration') - execute_curl('/tmp/curl_configuration', args.work_dir) - - -if __name__ == '__main__': - main() diff --git a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/debian-copyright-mirror.sh b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/debian-copyright-mirror.sh index a51fdfa08f6..561adea27d3 100755 --- a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/debian-copyright-mirror.sh +++ b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/debian-copyright-mirror.sh @@ -17,20 +17,13 @@ # # Maintain a GCS bucket mirror of Debian's copyright files # -# -# # Inputs: -# * A local work directory -# * GCS bucket name + path to tarball +# * A local work directory ($WORK_DIR) +# * GCS bucket name + path to tarball ($GCS_PATH) # -# Setting BE_VERBOSE to an empty string or null value suppresses silencing of -# commands - mkdir -p "${WORK_DIR}" || true -python3 debian-copyright-mirror.py "${WORK_DIR}/metadata.ftp-master.debian.org/changelogs/" - -tar -C "${WORK_DIR}" -cf "${WORK_DIR}/$(basename ${GCS_PATH})" . - -gcloud storage ${BE_VERBOSE="--quiet"} cp "${WORK_DIR}/$(basename ${GCS_PATH})" "${GCS_PATH}" +debian-copyright-mirror \ + -work-dir "${WORK_DIR}/metadata.ftp-master.debian.org/changelogs/" \ + -gcs-path "${GCS_PATH}" From b72b692149bb8fe817b3ad06985bdcf0f3e44f3c Mon Sep 17 00:00:00 2001 From: Jess Lowe Date: Wed, 19 Aug 2026 03:46:10 +0000 Subject: [PATCH 3/7] build and stage stuff --- deployment/build-and-stage.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deployment/build-and-stage.yaml b/deployment/build-and-stage.yaml index 522a39ac740..80645a1d7f3 100644 --- a/deployment/build-and-stage.yaml +++ b/deployment/build-and-stage.yaml @@ -368,8 +368,8 @@ steps: # Build/push Debian copyright mirror image - name: gcr.io/cloud-builders/docker - args: ['build', '-t', 'gcr.io/oss-vdb/debian-copyright-mirror:latest', '-t', 'gcr.io/oss-vdb/debian-copyright-mirror:$COMMIT_SHA', '.'] - dir: 'vulnfeeds/cmd/mirrors/debian-copyright-mirror' + args: ['build', '-t', 'gcr.io/oss-vdb/debian-copyright-mirror:latest', '-t', 'gcr.io/oss-vdb/debian-copyright-mirror:$COMMIT_SHA', '-f', 'cmd/mirrors/debian-copyright-mirror/Dockerfile', '.'] + dir: 'vulnfeeds' id: 'build-debian-copyright-mirror' waitFor: ['setup'] - name: gcr.io/cloud-builders/docker From 5074da28ac068679f8f094d37c859436a02bdabb Mon Sep 17 00:00:00 2001 From: Jess Lowe Date: Fri, 21 Aug 2026 03:09:03 +0000 Subject: [PATCH 4/7] no bash script, and work_dir nearby --- .../debian-copyright-mirror/Dockerfile | 9 +++--- .../mirrors/debian-copyright-mirror/README.md | 21 ++++++++----- .../debian-copyright-mirror.sh | 29 ----------------- .../mirrors/debian-copyright-mirror/main.go | 31 +++++++++++++++---- 4 files changed, 42 insertions(+), 48 deletions(-) delete mode 100755 vulnfeeds/cmd/mirrors/debian-copyright-mirror/debian-copyright-mirror.sh diff --git a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/Dockerfile b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/Dockerfile index 6ef0187b35d..2a9dc3025a4 100644 --- a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/Dockerfile +++ b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/Dockerfile @@ -24,11 +24,10 @@ RUN go mod download COPY ./ /src/ RUN CGO_ENABLED=0 go build -o debian-copyright-mirror ./cmd/mirrors/debian-copyright-mirror -FROM gcr.io/google.com/cloudsdktool/google-cloud-cli:alpine@sha256:be40864452bd6d7be21632a1dc18adf03d423de0bf2595f4a287c22400c484bb +FROM alpine:3.21 -RUN apk add --no-cache xz curl +RUN apk add --no-cache xz curl ca-certificates -COPY --from=GO_BUILD /src/debian-copyright-mirror /usr/local/bin/ -COPY ./cmd/mirrors/debian-copyright-mirror/debian-copyright-mirror.sh / +COPY --from=GO_BUILD /src/debian-copyright-mirror /usr/local/bin/debian-copyright-mirror -ENTRYPOINT ["/debian-copyright-mirror.sh"] +ENTRYPOINT ["/usr/local/bin/debian-copyright-mirror"] diff --git a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/README.md b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/README.md index eb139d03051..89a32d77893 100644 --- a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/README.md +++ b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/README.md @@ -70,29 +70,33 @@ go run ./cmd/mirrors/debian-copyright-mirror [flags] [work_dir] Examples: ```bash -# Download copyright files into /tmp/debian_copyright using 50 workers -go run ./cmd/mirrors/debian-copyright-mirror -workers 50 /tmp/debian_copyright +# Download copyright files into ./debian_copyright (relative to current directory) using 50 workers +go run ./cmd/mirrors/debian-copyright-mirror -workers 50 + +# Specify a custom relative output directory +go run ./cmd/mirrors/debian-copyright-mirror -out-dir debian_copyright # Incremental run skipping already downloaded files -go run ./cmd/mirrors/debian-copyright-mirror -skip-existing /tmp/debian_copyright +go run ./cmd/mirrors/debian-copyright-mirror -skip-existing # Download and create a local tar archive -go run ./cmd/mirrors/debian-copyright-mirror -tar-path /scratch/debian_copyright.tar /tmp/debian_copyright +go run ./cmd/mirrors/debian-copyright-mirror -tar-path debian_copyright.tar # Download and stream directly to GCS -go run ./cmd/mirrors/debian-copyright-mirror -gcs-path gs://my-bucket/debian_copyright.tar /tmp/debian_copyright +go run ./cmd/mirrors/debian-copyright-mirror -gcs-path gs://my-bucket/debian_copyright.tar ``` ### Flags | Flag | Type | Default | Description | | :--- | :--- | :--- | :--- | -| `-work-dir` | string | `.` | Target directory to save downloaded copyright files (can also be passed as the first positional argument). | +| `-out-dir` | string | `debian_copyright` | Target directory (relative or absolute) to save downloaded copyright files. | +| `-work-dir` | string | `""` | Alias for `-out-dir` (can also be passed as the first positional argument). | | `-workers` | int | `50` | Number of concurrent download workers. | | `-skip-existing` | bool | `false` | Skip downloading files that already exist on disk with non-zero size. | | `-max-failure-rate` | float | `0.05` | Maximum allowable fraction of failed downloads (e.g. `0.05` = 5%) before failing the job. | | `-gcs-path` | string | `""` | Destination GCS path for the tarball archive (e.g. `gs://bucket/debian_copyright.tar`). Defaults to `$GCS_PATH` if unset. | -| `-tar-path` | string | `""` | Optional local destination path for tarball archive. | +| `-tar-path` | string | `""` | Optional local destination path for tarball archive (e.g. `debian_copyright.tar`). | | `-filelist-url` | string | `https://metadata.ftp-master.debian.org/changelogs/filelist.yaml.xz` | URL of the Debian filelist YAML archive. | | `-url-base` | string | `https://metadata.ftp-master.debian.org/changelogs` | Base URL for downloading individual copyright files. | | `-prefix-filter` | string | `main/` | Archive section prefix to filter (e.g., `main/`). | @@ -101,6 +105,7 @@ go run ./cmd/mirrors/debian-copyright-mirror -gcs-path gs://my-bucket/debian_cop | `-curl-config-file` | string | `""` | Optional path to output the generated curl configuration file. | | `-curl-config-only` | bool | `false` | Generate the curl configuration file and exit immediately without downloading. | + --- ## Docker & Deployment @@ -118,4 +123,4 @@ docker build -t gcr.io/oss-vdb/debian-copyright-mirror:latest -f cmd/mirrors/deb In production, this mirror runs as a scheduled Kubernetes `CronJob` in GKE (defined in `deployment/clouddeploy/gke-workers/base/feeds/debian-copyright-mirror.yaml` and environment overlays): - **Schedule**: Runs daily (`0 5 * * *` Sydney time). -- **Entrypoint**: Executes `debian-copyright-mirror.sh`, which runs `debian-copyright-mirror` with `-gcs-path "${GCS_PATH}"` to stream the archive directly to `gs://cve-osv-conversion/debian_copyright/debian_copyright.tar`. +- **Entrypoint**: Runs the `debian-copyright-mirror` Go binary directly, which reads `WORK_DIR` (e.g. `/scratch`) and `GCS_PATH` (e.g. `gs://cve-osv-conversion/debian_copyright/debian_copyright.tar`) from the environment and streams the archive directly to Cloud Storage. diff --git a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/debian-copyright-mirror.sh b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/debian-copyright-mirror.sh deleted file mode 100755 index 561adea27d3..00000000000 --- a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/debian-copyright-mirror.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -# -# Copyright 2021 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# -# Maintain a GCS bucket mirror of Debian's copyright files -# -# Inputs: -# * A local work directory ($WORK_DIR) -# * GCS bucket name + path to tarball ($GCS_PATH) -# - -mkdir -p "${WORK_DIR}" || true - -debian-copyright-mirror \ - -work-dir "${WORK_DIR}/metadata.ftp-master.debian.org/changelogs/" \ - -gcs-path "${GCS_PATH}" diff --git a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main.go b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main.go index 19b314650f1..c698525b69b 100644 --- a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main.go +++ b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main.go @@ -281,28 +281,47 @@ func main() { logger.InitGlobalLogger() defer logger.Close() - workDirFlag := flag.String("work-dir", "", "Directory to download copyright files into.") + // Input & source configuration flags. filelistURL := flag.String("filelist-url", DefaultFilelistURL, "URL of the Debian filelist.yaml.xz file.") urlBase := flag.String("url-base", DefaultURLBase, "Base URL for downloading Debian changelog/copyright files.") prefixFilter := flag.String("prefix-filter", DefaultPrefixFilter, "Prefix filter for package paths to download (e.g. 'main/').") minExpectedFiles := flag.Int("min-expected-files", DefaultMinExpectedFiles, "Minimum expected number of copyright files.") + + // Local output directory & archive flags. + outDirFlag := flag.String("out-dir", "", "Directory to download copyright files into (defaults to WORK_DIR env var or ./debian_copyright).") + workDirFlag := flag.String("work-dir", "", "Alias for -out-dir.") + tarPath := flag.String("tar-path", "", "Optional local destination path for tarball archive (e.g. debian_copyright.tar).") + + // Concurrency & fault tolerance flags. numWorkers := flag.Int("workers", DefaultNumWorkers, "Number of concurrent download workers.") skipExisting := flag.Bool("skip-existing", false, "Skip downloading files that already exist on disk with non-zero size.") maxFailureRate := flag.Float64("max-failure-rate", DefaultMaxFailureRate, "Maximum allowable fraction of failed downloads before failing the job.") + + // GCS upload flags. gcsPath := flag.String("gcs-path", "", "Destination GCS path for tarball archive (e.g. gs://bucket/path/debian_copyright.tar). Defaults to GCS_PATH env var if unset.") - tarPath := flag.String("tar-path", "", "Optional local destination path for tarball archive (e.g. /scratch/debian_copyright.tar).") + + // Curl fallback flags. useCurl := flag.Bool("use-curl", false, "If true, delegate downloads to curl --parallel instead of Go HTTP workers.") curlConfigFile := flag.String("curl-config-file", "", "Optional path to write the generated curl configuration to.") curlConfigOnly := flag.Bool("curl-config-only", false, "If true, only write the curl configuration file and exit.") flag.Parse() - workDir := *workDirFlag - if workDir == "" && flag.NArg() > 0 { + workDir := *outDirFlag + if *workDirFlag != "" { + workDir = *workDirFlag + } else if flag.NArg() > 0 { workDir = flag.Arg(0) + } else if workDir == "" { + if envWorkDir := os.Getenv("WORK_DIR"); envWorkDir != "" { + workDir = envWorkDir + } else { + workDir = "debian_copyright" + } } - if workDir == "" { - workDir = "." + workDir = filepath.Clean(workDir) + if err := os.MkdirAll(workDir, 0755); err != nil { + logger.Fatal("Failed to create work directory", slog.String("workDir", workDir), slog.Any("err", err)) } ctx := context.Background() From 84522a16bb14bf01989c781ce9fd9a898626bf1b Mon Sep 17 00:00:00 2001 From: Jess Lowe Date: Fri, 21 Aug 2026 03:12:14 +0000 Subject: [PATCH 5/7] fix lint --- vulnfeeds/cmd/mirrors/debian-copyright-mirror/main.go | 7 ++++++- vulnfeeds/cmd/mirrors/debian-copyright-mirror/main_test.go | 1 + vulnfeeds/gcs-tools/gcs.go | 1 + vulnfeeds/utility/archive.go | 5 ++++- vulnfeeds/utility/archive_test.go | 4 ++-- vulnfeeds/utility/decompress.go | 3 +++ vulnfeeds/utility/decompress_test.go | 4 ++-- vulnfeeds/utility/download.go | 1 + vulnfeeds/utility/download_test.go | 1 + 9 files changed, 21 insertions(+), 6 deletions(-) diff --git a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main.go b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main.go index c698525b69b..7be32bc1fba 100644 --- a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main.go +++ b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main.go @@ -64,6 +64,7 @@ func ExtractUnstableCopyright(r io.Reader, prefixFilter string) ([]string, error if len(line) > 0 && line[0] != ' ' && line[0] != '\t' && strings.HasSuffix(line, ":") { inUnstable = false currentPkgFound = false + continue } @@ -79,6 +80,7 @@ func ExtractUnstableCopyright(r io.Reader, prefixFilter string) ([]string, error } else { inUnstable = false } + continue } @@ -123,7 +125,7 @@ func DownloadFilesConcurrently(ctx context.Context, client *http.Client, filelis g, ctx := errgroup.WithContext(ctx) - for i := 0; i < numWorkers; i++ { + for range numWorkers { g.Go(func() error { for path := range jobs { select { @@ -140,6 +142,7 @@ func DownloadFilesConcurrently(ctx context.Context, client *http.Client, filelis skipped.Add(1) count := completed.Add(1) logProgress(count, total, startTime) + continue } } @@ -152,6 +155,7 @@ func DownloadFilesConcurrently(ctx context.Context, client *http.Client, filelis logProgress(count, total, startTime) } } + return nil }) } @@ -274,6 +278,7 @@ func UploadTarToGCS(ctx context.Context, storageClient *storage.Client, workDir, } logger.Info("Successfully uploaded tar archive to GCS", slog.String("gcsURI", gcsURI)) + return nil } diff --git a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main_test.go b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main_test.go index 2d1b39a974a..c120c844d2d 100644 --- a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main_test.go +++ b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main_test.go @@ -150,6 +150,7 @@ func TestDownloadFilesConcurrently(t *testing.T) { if content, ok := fileMap[path]; ok { w.WriteHeader(http.StatusOK) _, _ = fmt.Fprint(w, content) + return } w.WriteHeader(http.StatusNotFound) diff --git a/vulnfeeds/gcs-tools/gcs.go b/vulnfeeds/gcs-tools/gcs.go index deb217c17e3..7ccdd37a3a7 100644 --- a/vulnfeeds/gcs-tools/gcs.go +++ b/vulnfeeds/gcs-tools/gcs.go @@ -155,6 +155,7 @@ func ParseGCSPath(gcsURI string) (bucket, object string, err error) { if len(parts) < 2 || parts[0] == "" || parts[1] == "" { return "", "", fmt.Errorf("invalid GCS URI format %q (expected gs://bucket/object)", gcsURI) } + return parts[0], parts[1], nil } diff --git a/vulnfeeds/utility/archive.go b/vulnfeeds/utility/archive.go index 267b8e4ba4e..633acc24358 100644 --- a/vulnfeeds/utility/archive.go +++ b/vulnfeeds/utility/archive.go @@ -29,6 +29,7 @@ func CreateTarArchive(sourceDir string, w io.Writer) error { defer tw.Close() sourceDir = filepath.Clean(sourceDir) + return filepath.Walk(sourceDir, func(file string, fi os.FileInfo, err error) error { if err != nil { return err @@ -73,6 +74,8 @@ func CreateTarArchive(sourceDir string, w io.Writer) error { }) } +const maxTarEntrySize = 1 * 1024 * 1024 * 1024 // 1GB max entry size limit + // ExtractTarArchive extracts a tar archive stream into the destination directory. func ExtractTarArchive(r io.Reader, destDir string) error { tr := tar.NewReader(r) @@ -106,7 +109,7 @@ func ExtractTarArchive(r io.Reader, destDir string) error { if err != nil { return fmt.Errorf("failed to create file %s: %w", targetPath, err) } - if _, err := io.Copy(outFile, tr); err != nil { + if _, err := io.Copy(outFile, io.LimitReader(tr, maxTarEntrySize)); err != nil { outFile.Close() return fmt.Errorf("failed to write file content %s: %w", targetPath, err) } diff --git a/vulnfeeds/utility/archive_test.go b/vulnfeeds/utility/archive_test.go index a853f771bf1..a13a075b0c3 100644 --- a/vulnfeeds/utility/archive_test.go +++ b/vulnfeeds/utility/archive_test.go @@ -32,10 +32,10 @@ func TestTarArchiveRoundtrip(t *testing.T) { file1 := filepath.Join(sourceDir, "file1.txt") file2 := filepath.Join(sourceDir, "sub", "file2.txt") - if err := os.WriteFile(file1, []byte("hello archive"), 0644); err != nil { + if err := os.WriteFile(file1, []byte("hello archive"), 0600); err != nil { t.Fatalf("Failed to write file1: %v", err) } - if err := os.WriteFile(file2, []byte("nested file content"), 0644); err != nil { + if err := os.WriteFile(file2, []byte("nested file content"), 0600); err != nil { t.Fatalf("Failed to write file2: %v", err) } diff --git a/vulnfeeds/utility/decompress.go b/vulnfeeds/utility/decompress.go index 8f80ea2d0f3..32080ab6cb8 100644 --- a/vulnfeeds/utility/decompress.go +++ b/vulnfeeds/utility/decompress.go @@ -28,6 +28,7 @@ import ( // is waited for and input resources are closed when Close() is called. type SubprocessReadCloser struct { io.ReadCloser + cmd *exec.Cmd bodyToClose io.Closer } @@ -53,6 +54,7 @@ func (s *SubprocessReadCloser) Close() error { } } } + return errors.Join(errs...) } @@ -63,6 +65,7 @@ func getXZDecompressCommand() (string, error) { if _, err := exec.LookPath("unxz"); err == nil { return "unxz", nil } + return "", errors.New("neither 'xz' nor 'unxz' command-line tool found in PATH") } diff --git a/vulnfeeds/utility/decompress_test.go b/vulnfeeds/utility/decompress_test.go index fe404b02da1..e66ca858e8e 100644 --- a/vulnfeeds/utility/decompress_test.go +++ b/vulnfeeds/utility/decompress_test.go @@ -58,7 +58,7 @@ func TestDecompressXZ(t *testing.T) { func TestDecompressXZFile(t *testing.T) { tempDir := t.TempDir() filePath := filepath.Join(tempDir, "test.xz") - if err := os.WriteFile(filePath, testXZPayload, 0644); err != nil { + if err := os.WriteFile(filePath, testXZPayload, 0600); err != nil { t.Fatalf("Failed to write test file: %v", err) } @@ -81,7 +81,7 @@ func TestDecompressXZFile(t *testing.T) { } func TestFetchAndDecompressXZ(t *testing.T) { - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) _, _ = w.Write(testXZPayload) })) diff --git a/vulnfeeds/utility/download.go b/vulnfeeds/utility/download.go index 1b2bed79635..df38ee6725e 100644 --- a/vulnfeeds/utility/download.go +++ b/vulnfeeds/utility/download.go @@ -62,6 +62,7 @@ func DownloadFile(ctx context.Context, client *http.Client, url, destPath string if resp.StatusCode >= 500 || resp.StatusCode == http.StatusTooManyRequests { return retry.RetryableError(fmt.Errorf("HTTP status %d for %s", resp.StatusCode, url)) } + return fmt.Errorf("HTTP status %d for %s", resp.StatusCode, url) } diff --git a/vulnfeeds/utility/download_test.go b/vulnfeeds/utility/download_test.go index d1eb62b48e4..417e62f361b 100644 --- a/vulnfeeds/utility/download_test.go +++ b/vulnfeeds/utility/download_test.go @@ -29,6 +29,7 @@ func TestDownloadFile(t *testing.T) { if r.URL.Path == "/file.txt" { w.WriteHeader(http.StatusOK) _, _ = fmt.Fprint(w, "downloaded content") + return } w.WriteHeader(http.StatusNotFound) From 9b6cd2fa966fd5bc45935e621d77c02c92b278dd Mon Sep 17 00:00:00 2001 From: Jess Lowe Date: Fri, 21 Aug 2026 03:35:34 +0000 Subject: [PATCH 6/7] prevent zip slip --- vulnfeeds/utility/archive.go | 67 ++++++++++++++++++++++++++----- vulnfeeds/utility/archive_test.go | 39 ++++++++++++++++++ 2 files changed, 96 insertions(+), 10 deletions(-) diff --git a/vulnfeeds/utility/archive.go b/vulnfeeds/utility/archive.go index 633acc24358..34aad69f075 100644 --- a/vulnfeeds/utility/archive.go +++ b/vulnfeeds/utility/archive.go @@ -16,6 +16,7 @@ package utility import ( "archive/tar" + "errors" "fmt" "io" "os" @@ -23,6 +24,8 @@ import ( "strings" ) +const maxTarEntrySize = 1 * 1024 * 1024 * 1024 // 1GB max entry size limit for decompression bomb protection + // CreateTarArchive archives all files and subdirectories in sourceDir into the tar writer. func CreateTarArchive(sourceDir string, w io.Writer) error { tw := tar.NewWriter(w) @@ -74,38 +77,77 @@ func CreateTarArchive(sourceDir string, w io.Writer) error { }) } -const maxTarEntrySize = 1 * 1024 * 1024 * 1024 // 1GB max entry size limit +// sanitizeTarPath validates that an archive entry name does not escape destDir (Zip Slip protection) +// and returns the canonical target path. +func sanitizeTarPath(destDir, name string) (string, error) { + cleanDest := filepath.Clean(destDir) + cleanName := filepath.Clean(filepath.FromSlash(name)) + + // Reject absolute paths and parent directory traversals + if filepath.IsAbs(cleanName) || strings.HasPrefix(cleanName, string(os.PathSeparator)) { + return "", fmt.Errorf("illegal absolute path in tar archive: %s", name) + } + if cleanName == ".." || strings.HasPrefix(cleanName, ".."+string(os.PathSeparator)) { + return "", fmt.Errorf("path traversal attempt in tar archive: %s", name) + } + + targetPath := filepath.Join(cleanDest, cleanName) + cleanTarget := filepath.Clean(targetPath) + + // Ensure target path is strictly inside cleanDest + rel, err := filepath.Rel(cleanDest, cleanTarget) + if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(os.PathSeparator)) { + return "", fmt.Errorf("path traversal attempt in tar archive: %s", name) + } + + return cleanTarget, nil +} -// ExtractTarArchive extracts a tar archive stream into the destination directory. +// ExtractTarArchive extracts a tar archive stream into the destination directory, +// protecting against path traversal (Zip Slip), symlink hijacking, and decompression bombs. func ExtractTarArchive(r io.Reader, destDir string) error { tr := tar.NewReader(r) cleanDest := filepath.Clean(destDir) + if err := os.MkdirAll(cleanDest, 0755); err != nil { + return fmt.Errorf("failed to create destination directory %s: %w", cleanDest, err) + } + for { hdr, err := tr.Next() - if err == io.EOF { + if errors.Is(err, io.EOF) { break } if err != nil { return fmt.Errorf("error reading tar entry: %w", err) } - // Prevent Zip Slip / path traversal - targetPath := filepath.Join(cleanDest, filepath.FromSlash(hdr.Name)) - if !strings.HasPrefix(targetPath, cleanDest+string(os.PathSeparator)) && targetPath != cleanDest { - return fmt.Errorf("illegal file path in tar archive: %s", hdr.Name) + targetPath, err := sanitizeTarPath(cleanDest, hdr.Name) + if err != nil { + return err } switch hdr.Typeflag { case tar.TypeDir: + if targetPath == cleanDest { + continue + } if err := os.MkdirAll(targetPath, 0755); err != nil { return fmt.Errorf("failed to create directory %s: %w", targetPath, err) } case tar.TypeReg: - if err := os.MkdirAll(filepath.Dir(targetPath), 0755); err != nil { + if targetPath == cleanDest { + return fmt.Errorf("tar entry attempts to overwrite destination directory: %s", hdr.Name) + } + dir := filepath.Dir(targetPath) + if err := os.MkdirAll(dir, 0755); err != nil { return fmt.Errorf("failed to create directory for file %s: %w", targetPath, err) } - outFile, err := os.OpenFile(targetPath, os.O_CREATE|os.O_RDWR|os.O_TRUNC, hdr.FileInfo().Mode()) + + // Clean existing file to avoid writing through pre-existing symlinks + _ = os.Remove(targetPath) + + outFile, err := os.OpenFile(targetPath, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0600) if err != nil { return fmt.Errorf("failed to create file %s: %w", targetPath, err) } @@ -113,7 +155,12 @@ func ExtractTarArchive(r io.Reader, destDir string) error { outFile.Close() return fmt.Errorf("failed to write file content %s: %w", targetPath, err) } - outFile.Close() + if err := outFile.Close(); err != nil { + return fmt.Errorf("failed to close file %s: %w", targetPath, err) + } + default: + // Skip unsupported entry types (symlinks, devices, pipes) to prevent link-based traversal attacks + continue } } diff --git a/vulnfeeds/utility/archive_test.go b/vulnfeeds/utility/archive_test.go index a13a075b0c3..f1e79c651f5 100644 --- a/vulnfeeds/utility/archive_test.go +++ b/vulnfeeds/utility/archive_test.go @@ -15,6 +15,7 @@ package utility import ( + "archive/tar" "bytes" "os" "path/filepath" @@ -67,3 +68,41 @@ func TestTarArchiveRoundtrip(t *testing.T) { t.Errorf("file2 content = %q, want %q", string(data2), "nested file content") } } + +func TestExtractTarArchive_ZipSlipProtection(t *testing.T) { + maliciousPaths := []string{ + "../escaped.txt", + "../../../../etc/passwd", + "/etc/passwd", + "sub/../../escaped.txt", + "sub/../../../escaped.txt", + "../dest_sibling.txt", + } + + for _, malPath := range maliciousPaths { + t.Run(malPath, func(t *testing.T) { + destDir := filepath.Join(t.TempDir(), "target") + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + + hdr := &tar.Header{ + Name: malPath, + Mode: 0600, + Size: int64(len("malicious")), + Typeflag: tar.TypeReg, + } + if err := tw.WriteHeader(hdr); err != nil { + t.Fatalf("Failed to write tar header: %v", err) + } + if _, err := tw.Write([]byte("malicious")); err != nil { + t.Fatalf("Failed to write tar body: %v", err) + } + tw.Close() + + err := ExtractTarArchive(&buf, destDir) + if err == nil { + t.Errorf("Expected Zip Slip error for path %q, got nil", malPath) + } + }) + } +} From 507414c65925e78fd4185f18b5bfb95b5c5acb65 Mon Sep 17 00:00:00 2001 From: Jess Lowe Date: Mon, 24 Aug 2026 05:36:17 +0000 Subject: [PATCH 7/7] less code --- .../debian-copyright-mirror/Dockerfile | 2 +- .../mirrors/debian-copyright-mirror/main.go | 316 ++++-------------- .../debian-copyright-mirror/main_test.go | 53 ++- vulnfeeds/utility/archive.go | 168 ---------- vulnfeeds/utility/archive_test.go | 108 ------ vulnfeeds/utility/decompress.go | 130 ------- vulnfeeds/utility/decompress_test.go | 108 ------ vulnfeeds/utility/download.go | 103 ------ vulnfeeds/utility/download_test.go | 67 ---- 9 files changed, 110 insertions(+), 945 deletions(-) delete mode 100644 vulnfeeds/utility/archive.go delete mode 100644 vulnfeeds/utility/archive_test.go delete mode 100644 vulnfeeds/utility/decompress.go delete mode 100644 vulnfeeds/utility/decompress_test.go delete mode 100644 vulnfeeds/utility/download.go delete mode 100644 vulnfeeds/utility/download_test.go diff --git a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/Dockerfile b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/Dockerfile index 2a9dc3025a4..4cfef7f4ecc 100644 --- a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/Dockerfile +++ b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/Dockerfile @@ -26,7 +26,7 @@ RUN CGO_ENABLED=0 go build -o debian-copyright-mirror ./cmd/mirrors/debian-copyr FROM alpine:3.21 -RUN apk add --no-cache xz curl ca-certificates +RUN apk add --no-cache xz curl tar ca-certificates COPY --from=GO_BUILD /src/debian-copyright-mirror /usr/local/bin/debian-copyright-mirror diff --git a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main.go b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main.go index 7be32bc1fba..09782dcb556 100644 --- a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main.go +++ b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main.go @@ -27,14 +27,10 @@ import ( "os/exec" "path/filepath" "strings" - "sync/atomic" - "time" "cloud.google.com/go/storage" gcs "github.com/google/osv.dev/vulnfeeds/gcs-tools" - "github.com/google/osv.dev/vulnfeeds/utility" "github.com/google/osv.dev/vulnfeeds/utility/logger" - "golang.org/x/sync/errgroup" ) const ( @@ -42,8 +38,6 @@ const ( DefaultURLBase = "https://metadata.ftp-master.debian.org/changelogs" DefaultPrefixFilter = "main/" DefaultMinExpectedFiles = 40000 - DefaultNumWorkers = 50 - DefaultMaxFailureRate = 0.05 // 5% maximum allowed failure rate ) // ExtractUnstableCopyright parses Debian changelogs filelist YAML content from a reader @@ -60,7 +54,6 @@ func ExtractUnstableCopyright(r io.Reader, prefixFilter string) ([]string, error for scanner.Scan() { line := scanner.Text() - // Top-level package key: starts without whitespace and ends with colon if len(line) > 0 && line[0] != ' ' && line[0] != '\t' && strings.HasSuffix(line, ":") { inUnstable = false currentPkgFound = false @@ -73,25 +66,16 @@ func ExtractUnstableCopyright(r io.Reader, prefixFilter string) ([]string, error continue } - // Check if this is a suite/version subkey under the package if strings.HasPrefix(line, " ") && !strings.HasPrefix(line, " -") && strings.HasSuffix(line, ":") { - if trimmed == "unstable:" || trimmed == "'unstable':" || trimmed == `"unstable":` { - inUnstable = true - } else { - inUnstable = false - } + inUnstable = trimmed == "unstable:" || trimmed == "'unstable':" || trimmed == `"unstable":` continue } - // If we are within the "unstable" section and haven't found a copyright file for this package yet if inUnstable && !currentPkgFound && strings.HasPrefix(trimmed, "- ") { - entry := strings.TrimSpace(strings.TrimPrefix(trimmed, "- ")) - entry = strings.Trim(entry, `"'`) - if strings.HasSuffix(entry, "unstable_copyright") { - if prefixFilter == "" || strings.HasPrefix(entry, prefixFilter) { - results = append(results, entry) - } + entry := strings.Trim(strings.TrimSpace(strings.TrimPrefix(trimmed, "- ")), `"'`) + if strings.HasSuffix(entry, "unstable_copyright") && (prefixFilter == "" || strings.HasPrefix(entry, prefixFilter)) { + results = append(results, entry) currentPkgFound = true } } @@ -104,110 +88,35 @@ func ExtractUnstableCopyright(r io.Reader, prefixFilter string) ([]string, error return results, nil } -// DownloadFilesConcurrently downloads all files in filelist using a channel-based worker pool. -func DownloadFilesConcurrently(ctx context.Context, client *http.Client, filelist []string, urlBase, workDir string, numWorkers int, skipExisting bool, maxFailureRate float64) error { - if numWorkers <= 0 { - numWorkers = DefaultNumWorkers +func fetchCopyrightFiles(ctx context.Context, filelistURL, prefixFilter string) ([]string, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, filelistURL, nil) + if err != nil { + return nil, err } - - total := len(filelist) - logger.Info("Starting concurrent download of copyright files", - slog.Int("total_files", total), - slog.Int("workers", numWorkers), - slog.Bool("skip_existing", skipExisting), - ) - - jobs := make(chan string, numWorkers*2) - var completed atomic.Int64 - var failed atomic.Int64 - var skipped atomic.Int64 - startTime := time.Now() - - g, ctx := errgroup.WithContext(ctx) - - for range numWorkers { - g.Go(func() error { - for path := range jobs { - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - - fullURL := fmt.Sprintf("%s/%s", strings.TrimRight(urlBase, "/"), strings.TrimLeft(path, "/")) - destPath := filepath.Join(workDir, path) - - if skipExisting { - if fi, err := os.Stat(destPath); err == nil && fi.Size() > 0 { - skipped.Add(1) - count := completed.Add(1) - logProgress(count, total, startTime) - - continue - } - } - - if err := utility.DownloadFile(ctx, client, fullURL, destPath, false); err != nil { - failed.Add(1) - logger.Warn("Failed to download copyright file", slog.String("url", fullURL), slog.Any("err", err)) - } else { - count := completed.Add(1) - logProgress(count, total, startTime) - } - } - - return nil - }) + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, err } - - go func() { - defer close(jobs) - for _, path := range filelist { - select { - case <-ctx.Done(): - return - case jobs <- path: - } - } - }() - - if err := g.Wait(); err != nil { - return fmt.Errorf("concurrent download aborted: %w", err) + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("HTTP %d fetching %s", resp.StatusCode, filelistURL) } - completedCount := completed.Load() - failedCount := failed.Load() - skippedCount := skipped.Load() - - logger.Info("Finished downloading copyright files", - slog.Int64("completed", completedCount), - slog.Int64("failed", failedCount), - slog.Int64("skipped", skippedCount), - slog.Int("total", total), - slog.Duration("elapsed", time.Since(startTime)), - ) - - if total > 0 { - failRatio := float64(failedCount) / float64(total) - if failRatio > maxFailureRate || (completedCount == 0 && total > 0) { - return fmt.Errorf("too many downloads failed: %d/%d (%.2f%% failed, max threshold is %.2f%%)", - failedCount, total, failRatio*100.0, maxFailureRate*100.0) - } + cmd := exec.CommandContext(ctx, "xz", "-dc") + cmd.Stdin = resp.Body + cmd.Stderr = os.Stderr + stdout, err := cmd.StdoutPipe() + if err != nil { + return nil, err + } + if err := cmd.Start(); err != nil { + return nil, err } - return nil -} + files, err := ExtractUnstableCopyright(stdout, prefixFilter) + _ = cmd.Wait() -func logProgress(count int64, total int, startTime time.Time) { - if count%5000 == 0 || count == int64(total) { - pct := float64(count) / float64(total) * 100.0 - logger.Info("Download progress", - slog.Int64("completed", count), - slog.Int("total", total), - slog.Float64("percent", pct), - slog.Duration("elapsed", time.Since(startTime)), - ) - } + return files, err } // GenerateCurlConfiguration generates a curl config file for parallel downloads. @@ -248,67 +157,45 @@ func ExecuteCurl(ctx context.Context, configPath, workDir string) error { return nil } -// UploadTarToGCS creates a tar archive of workDir and streams it directly to GCS. -func UploadTarToGCS(ctx context.Context, storageClient *storage.Client, workDir, gcsURI string) error { +// CreateTarArchive creates a tar archive of workDir using the system tar command. +func CreateTarArchive(ctx context.Context, workDir, tarPath string) error { + // #nosec G204 -- arguments are locally configured paths + cmd := exec.CommandContext(ctx, "tar", "-C", workDir, "--exclude="+filepath.Base(tarPath), "-cf", tarPath, ".") + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + logger.Info("Creating tar archive", slog.String("workDir", workDir), slog.String("tarPath", tarPath)) + + return cmd.Run() +} + +func uploadToGCS(ctx context.Context, tarPath, gcsURI string) error { bucketName, objectName, err := gcs.ParseGCSPath(gcsURI) if err != nil { return err } - - logger.Info("Streaming tar archive to GCS", slog.String("bucket", bucketName), slog.String("object", objectName), slog.String("workDir", workDir)) - bkt := storageClient.Bucket(bucketName) - - pipeReader, pipeWriter := io.Pipe() - errChan := make(chan error, 1) - - go func() { - err := utility.CreateTarArchive(workDir, pipeWriter) - _ = pipeWriter.CloseWithError(err) - errChan <- err - }() - - uploadErr := gcs.UploadToGCS(ctx, bkt, objectName, pipeReader, "application/x-tar", nil) - archiveErr := <-errChan - - if uploadErr != nil { - return fmt.Errorf("failed to upload tar to GCS %s: %w", gcsURI, uploadErr) - } - if archiveErr != nil { - return fmt.Errorf("failed to create tar archive: %w", archiveErr) + client, err := storage.NewClient(ctx) + if err != nil { + return fmt.Errorf("failed to create GCS client: %w", err) } + defer client.Close() - logger.Info("Successfully uploaded tar archive to GCS", slog.String("gcsURI", gcsURI)) + logger.Info("Uploading tar archive to GCS", slog.String("bucket", bucketName), slog.String("object", objectName)) - return nil + return gcs.UploadFile(ctx, client.Bucket(bucketName), objectName, tarPath) } func main() { logger.InitGlobalLogger() defer logger.Close() - // Input & source configuration flags. filelistURL := flag.String("filelist-url", DefaultFilelistURL, "URL of the Debian filelist.yaml.xz file.") urlBase := flag.String("url-base", DefaultURLBase, "Base URL for downloading Debian changelog/copyright files.") prefixFilter := flag.String("prefix-filter", DefaultPrefixFilter, "Prefix filter for package paths to download (e.g. 'main/').") minExpectedFiles := flag.Int("min-expected-files", DefaultMinExpectedFiles, "Minimum expected number of copyright files.") - - // Local output directory & archive flags. outDirFlag := flag.String("out-dir", "", "Directory to download copyright files into (defaults to WORK_DIR env var or ./debian_copyright).") workDirFlag := flag.String("work-dir", "", "Alias for -out-dir.") - tarPath := flag.String("tar-path", "", "Optional local destination path for tarball archive (e.g. debian_copyright.tar).") - - // Concurrency & fault tolerance flags. - numWorkers := flag.Int("workers", DefaultNumWorkers, "Number of concurrent download workers.") - skipExisting := flag.Bool("skip-existing", false, "Skip downloading files that already exist on disk with non-zero size.") - maxFailureRate := flag.Float64("max-failure-rate", DefaultMaxFailureRate, "Maximum allowable fraction of failed downloads before failing the job.") - - // GCS upload flags. - gcsPath := flag.String("gcs-path", "", "Destination GCS path for tarball archive (e.g. gs://bucket/path/debian_copyright.tar). Defaults to GCS_PATH env var if unset.") - - // Curl fallback flags. - useCurl := flag.Bool("use-curl", false, "If true, delegate downloads to curl --parallel instead of Go HTTP workers.") - curlConfigFile := flag.String("curl-config-file", "", "Optional path to write the generated curl configuration to.") - curlConfigOnly := flag.Bool("curl-config-only", false, "If true, only write the curl configuration file and exit.") + tarPathFlag := flag.String("tar-path", "", "Optional local destination path for tarball archive.") + gcsPathFlag := flag.String("gcs-path", "", "Destination GCS path for tarball archive (defaults to GCS_PATH env var).") flag.Parse() @@ -325,42 +212,16 @@ func main() { } } workDir = filepath.Clean(workDir) - if err := os.MkdirAll(workDir, 0755); err != nil { - logger.Fatal("Failed to create work directory", slog.String("workDir", workDir), slog.Any("err", err)) - } ctx := context.Background() - poolSize := *numWorkers * 2 - if poolSize < 100 { - poolSize = 100 - } - - httpClient := &http.Client{ - Transport: &http.Transport{ - MaxIdleConns: poolSize, - MaxIdleConnsPerHost: poolSize, - IdleConnTimeout: 90 * time.Second, - ForceAttemptHTTP2: true, - }, - } - - logger.Info("Streaming and decompressing filelist", slog.String("url", *filelistURL)) - decompressedReader, err := utility.FetchAndDecompressXZ(ctx, httpClient, *filelistURL) + logger.Info("Fetching and extracting copyright filelist", slog.String("url", *filelistURL)) + copyrightFiles, err := fetchCopyrightFiles(ctx, *filelistURL, *prefixFilter) if err != nil { - logger.Fatal("Failed to fetch and decompress filelist", slog.Any("err", err)) + logger.Fatal("Failed to obtain copyright file list", slog.Any("err", err)) } - defer decompressedReader.Close() - - logger.Info("Extracting unstable copyright file paths", slog.String("prefixFilter", *prefixFilter)) - copyrightFiles, err := ExtractUnstableCopyright(decompressedReader, *prefixFilter) - if err != nil { - logger.Fatal("Failed to extract unstable copyright paths", slog.Any("err", err)) - } - _ = decompressedReader.Close() logger.Info("Discovered copyright files", slog.Int("count", len(copyrightFiles))) - if len(copyrightFiles) < *minExpectedFiles { logger.Fatal("Unexpectedly small number of copyright files found", slog.Int("found", len(copyrightFiles)), @@ -368,70 +229,39 @@ func main() { ) } - if *useCurl || *curlConfigOnly || *curlConfigFile != "" { - cfgPath := *curlConfigFile - if cfgPath == "" { - tempDir, err := os.MkdirTemp("", "debian-copyright-mirror-curl-*") - if err != nil { - logger.Fatal("Failed to create temporary directory for curl config", slog.Any("err", err)) - } - defer os.RemoveAll(tempDir) - cfgPath = filepath.Join(tempDir, "curl_configuration") - } - - logger.Info("Generating curl configuration", slog.String("path", cfgPath)) - if err := GenerateCurlConfiguration(copyrightFiles, *urlBase, cfgPath); err != nil { - logger.Fatal("Failed to generate curl configuration", slog.Any("err", err)) - } - - if *curlConfigOnly { - logger.Info("Curl configuration generated successfully; exiting as requested.") - return - } + tempDir, err := os.MkdirTemp("", "debian-copyright-mirror-*") + if err != nil { + logger.Fatal("Failed to create temp dir", slog.Any("err", err)) + } + defer os.RemoveAll(tempDir) - if *useCurl { - if err := ExecuteCurl(ctx, cfgPath, workDir); err != nil { - logger.Fatal("Curl download failed", slog.Any("err", err)) - } - } - } else { - if err := DownloadFilesConcurrently(ctx, httpClient, copyrightFiles, *urlBase, workDir, *numWorkers, *skipExisting, *maxFailureRate); err != nil { - logger.Fatal("Concurrent download failed", slog.Any("err", err)) - } + cfgPath := filepath.Join(tempDir, "curl_configuration") + if err := GenerateCurlConfiguration(copyrightFiles, *urlBase, cfgPath); err != nil { + logger.Fatal("Failed to generate curl configuration", slog.Any("err", err)) } - if *tarPath != "" { - logger.Info("Creating local tar archive", slog.String("path", *tarPath), slog.String("workDir", workDir)) - if err := os.MkdirAll(filepath.Dir(*tarPath), 0755); err != nil { - logger.Fatal("Failed to create tar destination directory", slog.Any("err", err)) - } - tarFile, err := os.Create(*tarPath) - if err != nil { - logger.Fatal("Failed to create tar file", slog.Any("err", err)) - } - if err := utility.CreateTarArchive(workDir, tarFile); err != nil { - _ = tarFile.Close() - logger.Fatal("Failed to write tar archive", slog.Any("err", err)) - } - if err := tarFile.Close(); err != nil { - logger.Fatal("Failed to close tar file", slog.Any("err", err)) - } - logger.Info("Successfully created local tar archive", slog.String("path", *tarPath)) + if err := ExecuteCurl(ctx, cfgPath, workDir); err != nil { + logger.Fatal("Curl download failed", slog.Any("err", err)) } - gcsDest := *gcsPath + gcsDest := *gcsPathFlag if gcsDest == "" { gcsDest = os.Getenv("GCS_PATH") } - if gcsDest != "" { - storageClient, err := storage.NewClient(ctx) - if err != nil { - logger.Fatal("Failed to create GCS client", slog.Any("err", err)) - } - defer storageClient.Close() - if err := UploadTarToGCS(ctx, storageClient, workDir, gcsDest); err != nil { - logger.Fatal("Failed to upload tar archive to GCS", slog.Any("err", err)) + tarDest := *tarPathFlag + if tarDest == "" && gcsDest != "" { + tarDest = filepath.Join(workDir, filepath.Base(gcsDest)) + } + + if tarDest != "" { + if err := CreateTarArchive(ctx, workDir, tarDest); err != nil { + logger.Fatal("Failed to create tar archive", slog.Any("err", err)) + } + if gcsDest != "" { + if err := uploadToGCS(ctx, tarDest, gcsDest); err != nil { + logger.Fatal("Failed to upload tar archive to GCS", slog.Any("err", err)) + } } } diff --git a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main_test.go b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main_test.go index c120c844d2d..f772ffff114 100644 --- a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main_test.go +++ b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main_test.go @@ -139,10 +139,10 @@ func TestGenerateCurlConfiguration(t *testing.T) { } } -func TestDownloadFilesConcurrently(t *testing.T) { +func TestExecuteCurl(t *testing.T) { fileMap := map[string]string{ - "main/a/pkg1/unstable_copyright": "Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/\nSource: https://github.com/foo/pkg1\n", - "main/b/pkg2/unstable_copyright": "Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/\nSource: https://github.com/bar/pkg2\n", + "main/a/pkg1/unstable_copyright": "Copyright pkg1\n", + "main/b/pkg2/unstable_copyright": "Copyright pkg2\n", } ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -158,37 +158,56 @@ func TestDownloadFilesConcurrently(t *testing.T) { defer ts.Close() tempDir := t.TempDir() - ctx := context.Background() - client := ts.Client() + configPath := filepath.Join(tempDir, "curl_config") + workDir := filepath.Join(tempDir, "work") filelist := []string{ "main/a/pkg1/unstable_copyright", "main/b/pkg2/unstable_copyright", } - if err := DownloadFilesConcurrently(ctx, client, filelist, ts.URL, tempDir, 2, false, 0.05); err != nil { - t.Fatalf("DownloadFilesConcurrently failed: %v", err) + if err := GenerateCurlConfiguration(filelist, ts.URL, configPath); err != nil { + t.Fatalf("GenerateCurlConfiguration failed: %v", err) + } + + if err := ExecuteCurl(context.Background(), configPath, workDir); err != nil { + t.Fatalf("ExecuteCurl failed: %v", err) } for _, relPath := range filelist { - fullPath := filepath.Join(tempDir, relPath) - data, err := os.ReadFile(fullPath) + data, err := os.ReadFile(filepath.Join(workDir, relPath)) if err != nil { - t.Errorf("Failed to read %s: %v", relPath, err) + t.Errorf("Failed to read downloaded file %s: %v", relPath, err) continue } if string(data) != fileMap[relPath] { t.Errorf("Content mismatch for %s: got %q, want %q", relPath, string(data), fileMap[relPath]) } } +} - // Test failure threshold breach - badFileList := []string{ - "main/missing/1", - "main/missing/2", +func TestCreateTarArchive(t *testing.T) { + tempDir := t.TempDir() + workDir := filepath.Join(tempDir, "work") + subDir := filepath.Join(workDir, "main", "a", "pkg1") + if err := os.MkdirAll(subDir, 0755); err != nil { + t.Fatalf("Failed to create test directory: %v", err) + } + filePath := filepath.Join(subDir, "unstable_copyright") + if err := os.WriteFile(filePath, []byte("test copyright content"), 0600); err != nil { + t.Fatalf("Failed to write test file: %v", err) + } + + tarPath := filepath.Join(workDir, "test_archive.tar") + if err := CreateTarArchive(context.Background(), workDir, tarPath); err != nil { + t.Fatalf("CreateTarArchive failed: %v", err) + } + + fi, err := os.Stat(tarPath) + if err != nil { + t.Fatalf("Expected tar file to exist: %v", err) } - err := DownloadFilesConcurrently(ctx, client, badFileList, ts.URL, tempDir, 2, false, 0.05) - if err == nil { - t.Errorf("Expected failure threshold error, got nil") + if fi.Size() == 0 { + t.Errorf("Expected non-empty tar archive") } } diff --git a/vulnfeeds/utility/archive.go b/vulnfeeds/utility/archive.go deleted file mode 100644 index 34aad69f075..00000000000 --- a/vulnfeeds/utility/archive.go +++ /dev/null @@ -1,168 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package utility - -import ( - "archive/tar" - "errors" - "fmt" - "io" - "os" - "path/filepath" - "strings" -) - -const maxTarEntrySize = 1 * 1024 * 1024 * 1024 // 1GB max entry size limit for decompression bomb protection - -// CreateTarArchive archives all files and subdirectories in sourceDir into the tar writer. -func CreateTarArchive(sourceDir string, w io.Writer) error { - tw := tar.NewWriter(w) - defer tw.Close() - - sourceDir = filepath.Clean(sourceDir) - - return filepath.Walk(sourceDir, func(file string, fi os.FileInfo, err error) error { - if err != nil { - return err - } - if file == sourceDir { - return nil - } - - relPath, err := filepath.Rel(sourceDir, file) - if err != nil { - return fmt.Errorf("failed to get relative path for %s: %w", file, err) - } - - tarName := filepath.ToSlash(relPath) - if fi.IsDir() { - tarName += "/" - } - - hdr, err := tar.FileInfoHeader(fi, "") - if err != nil { - return fmt.Errorf("failed to create tar header for %s: %w", file, err) - } - hdr.Name = tarName - - if err := tw.WriteHeader(hdr); err != nil { - return fmt.Errorf("failed to write tar header for %s: %w", file, err) - } - - if fi.Mode().IsRegular() { - f, err := os.Open(file) - if err != nil { - return fmt.Errorf("failed to open file %s: %w", file, err) - } - defer f.Close() - - if _, err := io.Copy(tw, f); err != nil { - return fmt.Errorf("failed to copy file %s to tar: %w", file, err) - } - } - - return nil - }) -} - -// sanitizeTarPath validates that an archive entry name does not escape destDir (Zip Slip protection) -// and returns the canonical target path. -func sanitizeTarPath(destDir, name string) (string, error) { - cleanDest := filepath.Clean(destDir) - cleanName := filepath.Clean(filepath.FromSlash(name)) - - // Reject absolute paths and parent directory traversals - if filepath.IsAbs(cleanName) || strings.HasPrefix(cleanName, string(os.PathSeparator)) { - return "", fmt.Errorf("illegal absolute path in tar archive: %s", name) - } - if cleanName == ".." || strings.HasPrefix(cleanName, ".."+string(os.PathSeparator)) { - return "", fmt.Errorf("path traversal attempt in tar archive: %s", name) - } - - targetPath := filepath.Join(cleanDest, cleanName) - cleanTarget := filepath.Clean(targetPath) - - // Ensure target path is strictly inside cleanDest - rel, err := filepath.Rel(cleanDest, cleanTarget) - if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(os.PathSeparator)) { - return "", fmt.Errorf("path traversal attempt in tar archive: %s", name) - } - - return cleanTarget, nil -} - -// ExtractTarArchive extracts a tar archive stream into the destination directory, -// protecting against path traversal (Zip Slip), symlink hijacking, and decompression bombs. -func ExtractTarArchive(r io.Reader, destDir string) error { - tr := tar.NewReader(r) - cleanDest := filepath.Clean(destDir) - - if err := os.MkdirAll(cleanDest, 0755); err != nil { - return fmt.Errorf("failed to create destination directory %s: %w", cleanDest, err) - } - - for { - hdr, err := tr.Next() - if errors.Is(err, io.EOF) { - break - } - if err != nil { - return fmt.Errorf("error reading tar entry: %w", err) - } - - targetPath, err := sanitizeTarPath(cleanDest, hdr.Name) - if err != nil { - return err - } - - switch hdr.Typeflag { - case tar.TypeDir: - if targetPath == cleanDest { - continue - } - if err := os.MkdirAll(targetPath, 0755); err != nil { - return fmt.Errorf("failed to create directory %s: %w", targetPath, err) - } - case tar.TypeReg: - if targetPath == cleanDest { - return fmt.Errorf("tar entry attempts to overwrite destination directory: %s", hdr.Name) - } - dir := filepath.Dir(targetPath) - if err := os.MkdirAll(dir, 0755); err != nil { - return fmt.Errorf("failed to create directory for file %s: %w", targetPath, err) - } - - // Clean existing file to avoid writing through pre-existing symlinks - _ = os.Remove(targetPath) - - outFile, err := os.OpenFile(targetPath, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0600) - if err != nil { - return fmt.Errorf("failed to create file %s: %w", targetPath, err) - } - if _, err := io.Copy(outFile, io.LimitReader(tr, maxTarEntrySize)); err != nil { - outFile.Close() - return fmt.Errorf("failed to write file content %s: %w", targetPath, err) - } - if err := outFile.Close(); err != nil { - return fmt.Errorf("failed to close file %s: %w", targetPath, err) - } - default: - // Skip unsupported entry types (symlinks, devices, pipes) to prevent link-based traversal attacks - continue - } - } - - return nil -} diff --git a/vulnfeeds/utility/archive_test.go b/vulnfeeds/utility/archive_test.go deleted file mode 100644 index f1e79c651f5..00000000000 --- a/vulnfeeds/utility/archive_test.go +++ /dev/null @@ -1,108 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package utility - -import ( - "archive/tar" - "bytes" - "os" - "path/filepath" - "testing" -) - -func TestTarArchiveRoundtrip(t *testing.T) { - tempDir := t.TempDir() - sourceDir := filepath.Join(tempDir, "source") - destDir := filepath.Join(tempDir, "dest") - - if err := os.MkdirAll(filepath.Join(sourceDir, "sub"), 0755); err != nil { - t.Fatalf("Failed to create test source dir: %v", err) - } - - file1 := filepath.Join(sourceDir, "file1.txt") - file2 := filepath.Join(sourceDir, "sub", "file2.txt") - if err := os.WriteFile(file1, []byte("hello archive"), 0600); err != nil { - t.Fatalf("Failed to write file1: %v", err) - } - if err := os.WriteFile(file2, []byte("nested file content"), 0600); err != nil { - t.Fatalf("Failed to write file2: %v", err) - } - - var buf bytes.Buffer - if err := CreateTarArchive(sourceDir, &buf); err != nil { - t.Fatalf("CreateTarArchive failed: %v", err) - } - - if err := ExtractTarArchive(&buf, destDir); err != nil { - t.Fatalf("ExtractTarArchive failed: %v", err) - } - - destFile1 := filepath.Join(destDir, "file1.txt") - destFile2 := filepath.Join(destDir, "sub", "file2.txt") - - data1, err := os.ReadFile(destFile1) - if err != nil { - t.Fatalf("Failed to read extracted file1: %v", err) - } - if string(data1) != "hello archive" { - t.Errorf("file1 content = %q, want %q", string(data1), "hello archive") - } - - data2, err := os.ReadFile(destFile2) - if err != nil { - t.Fatalf("Failed to read extracted file2: %v", err) - } - if string(data2) != "nested file content" { - t.Errorf("file2 content = %q, want %q", string(data2), "nested file content") - } -} - -func TestExtractTarArchive_ZipSlipProtection(t *testing.T) { - maliciousPaths := []string{ - "../escaped.txt", - "../../../../etc/passwd", - "/etc/passwd", - "sub/../../escaped.txt", - "sub/../../../escaped.txt", - "../dest_sibling.txt", - } - - for _, malPath := range maliciousPaths { - t.Run(malPath, func(t *testing.T) { - destDir := filepath.Join(t.TempDir(), "target") - var buf bytes.Buffer - tw := tar.NewWriter(&buf) - - hdr := &tar.Header{ - Name: malPath, - Mode: 0600, - Size: int64(len("malicious")), - Typeflag: tar.TypeReg, - } - if err := tw.WriteHeader(hdr); err != nil { - t.Fatalf("Failed to write tar header: %v", err) - } - if _, err := tw.Write([]byte("malicious")); err != nil { - t.Fatalf("Failed to write tar body: %v", err) - } - tw.Close() - - err := ExtractTarArchive(&buf, destDir) - if err == nil { - t.Errorf("Expected Zip Slip error for path %q, got nil", malPath) - } - }) - } -} diff --git a/vulnfeeds/utility/decompress.go b/vulnfeeds/utility/decompress.go deleted file mode 100644 index 32080ab6cb8..00000000000 --- a/vulnfeeds/utility/decompress.go +++ /dev/null @@ -1,130 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package utility - -import ( - "context" - "errors" - "fmt" - "io" - "net/http" - "os" - "os/exec" -) - -// SubprocessReadCloser wraps a command's stdout pipe and ensures the subprocess -// is waited for and input resources are closed when Close() is called. -type SubprocessReadCloser struct { - io.ReadCloser - - cmd *exec.Cmd - bodyToClose io.Closer -} - -// Close closes the underlying reader pipe, any input reader, and waits for the subprocess. -func (s *SubprocessReadCloser) Close() error { - var errs []error - if s.ReadCloser != nil { - if err := s.ReadCloser.Close(); err != nil { - errs = append(errs, err) - } - } - if s.bodyToClose != nil { - if err := s.bodyToClose.Close(); err != nil { - errs = append(errs, err) - } - } - if s.cmd != nil { - if err := s.cmd.Wait(); err != nil { - var exitErr *exec.ExitError - if errors.As(err, &exitErr) && !exitErr.Success() { - errs = append(errs, err) - } - } - } - - return errors.Join(errs...) -} - -func getXZDecompressCommand() (string, error) { - if _, err := exec.LookPath("xz"); err == nil { - return "xz", nil - } - if _, err := exec.LookPath("unxz"); err == nil { - return "unxz", nil - } - - return "", errors.New("neither 'xz' nor 'unxz' command-line tool found in PATH") -} - -// DecompressXZ streams an xz-compressed reader through the system xz or unxz command. -func DecompressXZ(ctx context.Context, r io.Reader) (io.ReadCloser, error) { - decompressCmd, err := getXZDecompressCommand() - if err != nil { - return nil, err - } - - cmd := exec.CommandContext(ctx, decompressCmd, "-dc") - cmd.Stdin = r - stdout, err := cmd.StdoutPipe() - if err != nil { - return nil, fmt.Errorf("failed to create stdout pipe for %s: %w", decompressCmd, err) - } - - if err := cmd.Start(); err != nil { - return nil, fmt.Errorf("failed to start %s: %w", decompressCmd, err) - } - - var bodyToClose io.Closer - if closer, ok := r.(io.Closer); ok { - bodyToClose = closer - } - - return &SubprocessReadCloser{ - ReadCloser: stdout, - cmd: cmd, - bodyToClose: bodyToClose, - }, nil -} - -// DecompressXZFile decompresses a local xz-compressed file. -func DecompressXZFile(ctx context.Context, xzFilePath string) (io.ReadCloser, error) { - f, err := os.Open(xzFilePath) - if err != nil { - return nil, fmt.Errorf("failed to open %s: %w", xzFilePath, err) - } - - return DecompressXZ(ctx, f) -} - -// FetchAndDecompressXZ fetches an xz-compressed URL over HTTP and streams -// its decompressed content directly through xz/unxz without saving to disk. -func FetchAndDecompressXZ(ctx context.Context, client *http.Client, url string) (io.ReadCloser, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) - if err != nil { - return nil, fmt.Errorf("failed to create request for %s: %w", url, err) - } - - resp, err := client.Do(req) - if err != nil { - return nil, fmt.Errorf("failed to fetch %s: %w", url, err) - } - if resp.StatusCode != http.StatusOK { - resp.Body.Close() - return nil, fmt.Errorf("HTTP status %d fetching %s", resp.StatusCode, url) - } - - return DecompressXZ(ctx, resp.Body) -} diff --git a/vulnfeeds/utility/decompress_test.go b/vulnfeeds/utility/decompress_test.go deleted file mode 100644 index e66ca858e8e..00000000000 --- a/vulnfeeds/utility/decompress_test.go +++ /dev/null @@ -1,108 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package utility - -import ( - "bytes" - "context" - "io" - "net/http" - "net/http/httptest" - "os" - "path/filepath" - "testing" -) - -var testXZPayload = []byte{ - 0xfd, 0x37, 0x7a, 0x58, 0x5a, 0x00, 0x00, 0x04, 0xe6, 0xd6, 0xb4, 0x46, - 0x02, 0x00, 0x21, 0x01, 0x16, 0x00, 0x00, 0x00, 0x74, 0x2f, 0xe5, 0xa3, - 0x01, 0x00, 0x1d, 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x44, 0x65, 0x62, - 0x69, 0x61, 0x6e, 0x20, 0x43, 0x6f, 0x70, 0x79, 0x72, 0x69, 0x67, 0x68, - 0x74, 0x20, 0x4d, 0x69, 0x72, 0x72, 0x6f, 0x72, 0x21, 0x00, 0x00, 0x00, - 0xc7, 0x2f, 0x01, 0x8a, 0x65, 0x5a, 0x9a, 0x97, 0x00, 0x01, 0x36, 0x1e, - 0x3d, 0x19, 0x95, 0x53, 0x1f, 0xb6, 0xf3, 0x7d, 0x01, 0x00, 0x00, 0x00, - 0x00, 0x04, 0x59, 0x5a, -} - -func TestDecompressXZ(t *testing.T) { - ctx := context.Background() - reader, err := DecompressXZ(ctx, bytes.NewReader(testXZPayload)) - if err != nil { - t.Fatalf("DecompressXZ failed: %v", err) - } - defer reader.Close() - - data, err := io.ReadAll(reader) - if err != nil { - t.Fatalf("Failed to read decompressed data: %v", err) - } - - want := "Hello Debian Copyright Mirror!" - if string(data) != want { - t.Errorf("Decompressed = %q, want %q", string(data), want) - } -} - -func TestDecompressXZFile(t *testing.T) { - tempDir := t.TempDir() - filePath := filepath.Join(tempDir, "test.xz") - if err := os.WriteFile(filePath, testXZPayload, 0600); err != nil { - t.Fatalf("Failed to write test file: %v", err) - } - - ctx := context.Background() - reader, err := DecompressXZFile(ctx, filePath) - if err != nil { - t.Fatalf("DecompressXZFile failed: %v", err) - } - defer reader.Close() - - data, err := io.ReadAll(reader) - if err != nil { - t.Fatalf("Failed to read decompressed data: %v", err) - } - - want := "Hello Debian Copyright Mirror!" - if string(data) != want { - t.Errorf("Decompressed = %q, want %q", string(data), want) - } -} - -func TestFetchAndDecompressXZ(t *testing.T) { - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.WriteHeader(http.StatusOK) - _, _ = w.Write(testXZPayload) - })) - defer ts.Close() - - ctx := context.Background() - client := ts.Client() - - reader, err := FetchAndDecompressXZ(ctx, client, ts.URL+"/file.xz") - if err != nil { - t.Fatalf("FetchAndDecompressXZ failed: %v", err) - } - defer reader.Close() - - data, err := io.ReadAll(reader) - if err != nil { - t.Fatalf("Failed to read decompressed data: %v", err) - } - - want := "Hello Debian Copyright Mirror!" - if string(data) != want { - t.Errorf("Decompressed = %q, want %q", string(data), want) - } -} diff --git a/vulnfeeds/utility/download.go b/vulnfeeds/utility/download.go deleted file mode 100644 index df38ee6725e..00000000000 --- a/vulnfeeds/utility/download.go +++ /dev/null @@ -1,103 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package utility - -import ( - "context" - "fmt" - "io" - "net/http" - "os" - "path/filepath" - "time" - - "github.com/sethvargo/go-retry" -) - -const ( - DefaultDownloadTimeout = 30 * time.Second -) - -// DownloadFile downloads a URL to a local destination file with automatic retries, -// safe temporary file writing, and on-demand directory creation. -// If skipExisting is true and destPath exists with non-zero size, the download is skipped. -func DownloadFile(ctx context.Context, client *http.Client, url, destPath string, skipExisting bool) error { - if skipExisting { - if fi, err := os.Stat(destPath); err == nil && fi.Size() > 0 { - return nil - } - } - - backoff := retry.NewExponential(1 * time.Second) - backoff = retry.WithMaxRetries(3, backoff) - - return retry.Do(ctx, backoff, func(ctx context.Context) error { - reqCtx, cancel := context.WithTimeout(ctx, DefaultDownloadTimeout) - defer cancel() - - req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, url, nil) - if err != nil { - return fmt.Errorf("failed to create request for %s: %w", url, err) - } - - resp, err := client.Do(req) - if err != nil { - return retry.RetryableError(fmt.Errorf("HTTP request failed for %s: %w", url, err)) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - if resp.StatusCode >= 500 || resp.StatusCode == http.StatusTooManyRequests { - return retry.RetryableError(fmt.Errorf("HTTP status %d for %s", resp.StatusCode, url)) - } - - return fmt.Errorf("HTTP status %d for %s", resp.StatusCode, url) - } - - dir := filepath.Dir(destPath) - tmpFile, err := os.CreateTemp(dir, "download-*") - if err != nil { - if os.IsNotExist(err) { - if mkdirErr := os.MkdirAll(dir, 0755); mkdirErr != nil { - return fmt.Errorf("failed to create directory %s: %w", dir, mkdirErr) - } - tmpFile, err = os.CreateTemp(dir, "download-*") - } - if err != nil { - return fmt.Errorf("failed to create temp file in %s: %w", dir, err) - } - } - tmpPath := tmpFile.Name() - - _, copyErr := io.Copy(tmpFile, resp.Body) - closeErr := tmpFile.Close() - - if copyErr != nil { - _ = os.Remove(tmpPath) - return retry.RetryableError(fmt.Errorf("failed to write data: %w", copyErr)) - } - if closeErr != nil { - _ = os.Remove(tmpPath) - return fmt.Errorf("failed to close temp file: %w", closeErr) - } - - if err := os.Rename(tmpPath, destPath); err != nil { - _ = os.Remove(tmpPath) - return fmt.Errorf("failed to move temp file to %s: %w", destPath, err) - } - - return nil - }) -} diff --git a/vulnfeeds/utility/download_test.go b/vulnfeeds/utility/download_test.go deleted file mode 100644 index 417e62f361b..00000000000 --- a/vulnfeeds/utility/download_test.go +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package utility - -import ( - "context" - "fmt" - "net/http" - "net/http/httptest" - "os" - "path/filepath" - "testing" -) - -func TestDownloadFile(t *testing.T) { - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path == "/file.txt" { - w.WriteHeader(http.StatusOK) - _, _ = fmt.Fprint(w, "downloaded content") - - return - } - w.WriteHeader(http.StatusNotFound) - })) - defer ts.Close() - - tempDir := t.TempDir() - destPath := filepath.Join(tempDir, "nested", "dir", "file.txt") - - ctx := context.Background() - client := ts.Client() - - if err := DownloadFile(ctx, client, ts.URL+"/file.txt", destPath, false); err != nil { - t.Fatalf("DownloadFile failed: %v", err) - } - - data, err := os.ReadFile(destPath) - if err != nil { - t.Fatalf("Failed to read downloaded file: %v", err) - } - if string(data) != "downloaded content" { - t.Errorf("content = %q, want %q", string(data), "downloaded content") - } - - // Test skipExisting = true - if err := DownloadFile(ctx, client, ts.URL+"/404-should-skip", destPath, true); err != nil { - t.Fatalf("DownloadFile with skipExisting should have succeeded without error, got %v", err) - } - - // Test 404 - err = DownloadFile(ctx, client, ts.URL+"/nonexistent", filepath.Join(tempDir, "404.txt"), false) - if err == nil { - t.Errorf("Expected error for 404, got nil") - } -}