diff --git a/deployment/build-and-stage.yaml b/deployment/build-and-stage.yaml index 529916e5dd4..2ef281f458e 100644 --- a/deployment/build-and-stage.yaml +++ b/deployment/build-and-stage.yaml @@ -373,8 +373,8 @@ steps: # Build/push Debian copyright mirror image - name: gcr.io/cloud-builders/docker - args: ['build', '-t', 'gcr.io/oss-vdb/debian-copyright-mirror:latest', '-t', 'gcr.io/oss-vdb/debian-copyright-mirror:$COMMIT_SHA', '.'] - dir: 'vulnfeeds/cmd/mirrors/debian-copyright-mirror' + args: ['build', '-t', 'gcr.io/oss-vdb/debian-copyright-mirror:latest', '-t', 'gcr.io/oss-vdb/debian-copyright-mirror:$COMMIT_SHA', '-f', 'cmd/mirrors/debian-copyright-mirror/Dockerfile', '.'] + dir: 'vulnfeeds' id: 'build-debian-copyright-mirror' waitFor: ['setup'] - name: gcr.io/cloud-builders/docker diff --git a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/Dockerfile b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/Dockerfile index 09afd30ba3f..4cfef7f4ecc 100644 --- a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/Dockerfile +++ b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/Dockerfile @@ -12,11 +12,22 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM gcr.io/google.com/cloudsdktool/google-cloud-cli:alpine@sha256:be40864452bd6d7be21632a1dc18adf03d423de0bf2595f4a287c22400c484bb +FROM golang:1.26.5-alpine@sha256:0178a641fbb4858c5f1b48e34bdaabe0350a330a1b1149aabd498d0699ff5fb2 AS GO_BUILD -RUN apk add py3-yaml +RUN mkdir /src +WORKDIR /src -COPY ./debian-copyright-mirror.sh / -COPY ./debian-copyright-mirror.py / +COPY ./go.mod /src/go.mod +COPY ./go.sum /src/go.sum +RUN go mod download -ENTRYPOINT ["/debian-copyright-mirror.sh"] +COPY ./ /src/ +RUN CGO_ENABLED=0 go build -o debian-copyright-mirror ./cmd/mirrors/debian-copyright-mirror + +FROM alpine:3.21 + +RUN apk add --no-cache xz curl tar ca-certificates + +COPY --from=GO_BUILD /src/debian-copyright-mirror /usr/local/bin/debian-copyright-mirror + +ENTRYPOINT ["/usr/local/bin/debian-copyright-mirror"] diff --git a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/README.md b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/README.md new file mode 100644 index 00000000000..89a32d77893 --- /dev/null +++ b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/README.md @@ -0,0 +1,126 @@ +# Debian Copyright Mirror + +`debian-copyright-mirror` maintains a mirror of machine-readable Debian package copyright files for packages in Debian `unstable` (`main` section). + +--- + +## Purpose & Usage in OSV + +In the OSV and vulnerability feeds pipeline, correlating Common Vulnerabilities and Exposures (CVE) entries and Common Platform Enumeration (CPE) identifiers with upstream open-source source code repositories (e.g., on GitHub, GitLab, or Git) is a crucial step for automated vulnerability management. + +### The Problem +- CVE entries and NVD CPE dictionaries often lack structured upstream repository URLs, or only provide arbitrary website references. +- Without a reliable mapping from package names and CPE products to source code repositories, automated tools cannot easily determine affected Git commits, tags, or version ranges for Git-based vulnerability matching. + +### How Debian Copyright Files Solve This +Debian packages in `unstable` follow the [Machine-readable debian/copyright format (DEP-5 / Copyright Specification 1.0)](https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/). These files contain structured metadata in header paragraphs, notably: + +```text +Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: example-lib +Source: https://github.com/example/example-lib +``` + +The `Source:` field provides a canonical, maintainer-verified URL to the upstream open-source repository. + +### Downstream Consumers +1. **`cpe-repo-gen`** (`cmd/mirrors/cpe-repo-gen`): + - Downloads the `debian_copyright.tar` archive generated by this tool from Google Cloud Storage (`gs://cve-osv-conversion/debian_copyright/debian_copyright.tar`). + - Scans the mirrored `unstable_copyright` files to resolve Debian package names and CPE products to upstream source code repositories. + - Produces the canonical `cpe_product_to_repo.json` mapping. +2. **OSV Conversion & Enrichment Pipelines**: + - Use the repository mappings derived from Debian copyright data to accurately calculate affected commit ranges, detect fixed revisions, and generate OSV-format records for Debian, NVD, and other feeds. + +--- + +## How It Works + +```mermaid +graph TD + A["Debian FTP Master
(filelist.yaml.xz)"] -->|HTTP Stream & xz -dc| B["debian-copyright-mirror
(Go binary)"] + B -->|Streaming YAML Parser| C["Extract ~44k+ main/
unstable_copyright paths"] + C -->|Channel-based Worker Pool| D["Download copyright files
to local directory"] + D -->|Validate failure rate & count| E["Native Tar Streaming"] + E -->|GCS Client Writer| F["GCS Bucket
(gs://cve-osv-conversion/...)"] + F -->|tar -xf| G["cpe-repo-gen
(CPE to Repo Mapping)"] +``` + +1. **Manifest Streaming & Decompression**: Fetches `filelist.yaml.xz` from `https://metadata.ftp-master.debian.org/changelogs/filelist.yaml.xz` over HTTP and streams it directly through `xz -dc` stdin without intermediate disk roundtrips. +2. **Manifest Parsing**: Parses the YAML manifest to discover all packages containing an `unstable_copyright` entry under the `unstable` suite in the `main` archive section. +3. **Validation**: Asserts that the number of discovered files meets a sanity threshold (by default, at least 40,000 files) to ensure upstream feeds were not corrupted or truncated. +4. **Optimized Concurrent Downloads**: + - Spawns a channel-based worker pool (default 50 workers) with HTTP/2 keep-alive connection reuse and exponential backoff retries. + - Optimized file and directory creation (avoids redundant `MkdirAll` calls on existing directories). + - Tracks success/failure metrics and enforces a maximum failure threshold (`-max-failure-rate`) to prevent false-positive completions on network or CDN outages. + - Supports incremental runs with `-skip-existing` to skip re-downloading unchanged files. + - Optional: Delegated download via `curl --parallel` using generated curl configuration files (`-use-curl`). +5. **Native Archival & GCS Streaming**: Packages the downloaded mirror into a `.tar` archive and streams it directly to Google Cloud Storage (e.g. `gs://cve-osv-conversion/debian_copyright/debian_copyright.tar`) using the Cloud Storage client. + +--- + +## CLI Usage + +### Running Locally + +```bash +# Build and run directly using Go +go run ./cmd/mirrors/debian-copyright-mirror [flags] [work_dir] +``` + +Examples: + +```bash +# Download copyright files into ./debian_copyright (relative to current directory) using 50 workers +go run ./cmd/mirrors/debian-copyright-mirror -workers 50 + +# Specify a custom relative output directory +go run ./cmd/mirrors/debian-copyright-mirror -out-dir debian_copyright + +# Incremental run skipping already downloaded files +go run ./cmd/mirrors/debian-copyright-mirror -skip-existing + +# Download and create a local tar archive +go run ./cmd/mirrors/debian-copyright-mirror -tar-path debian_copyright.tar + +# Download and stream directly to GCS +go run ./cmd/mirrors/debian-copyright-mirror -gcs-path gs://my-bucket/debian_copyright.tar +``` + +### Flags + +| Flag | Type | Default | Description | +| :--- | :--- | :--- | :--- | +| `-out-dir` | string | `debian_copyright` | Target directory (relative or absolute) to save downloaded copyright files. | +| `-work-dir` | string | `""` | Alias for `-out-dir` (can also be passed as the first positional argument). | +| `-workers` | int | `50` | Number of concurrent download workers. | +| `-skip-existing` | bool | `false` | Skip downloading files that already exist on disk with non-zero size. | +| `-max-failure-rate` | float | `0.05` | Maximum allowable fraction of failed downloads (e.g. `0.05` = 5%) before failing the job. | +| `-gcs-path` | string | `""` | Destination GCS path for the tarball archive (e.g. `gs://bucket/debian_copyright.tar`). Defaults to `$GCS_PATH` if unset. | +| `-tar-path` | string | `""` | Optional local destination path for tarball archive (e.g. `debian_copyright.tar`). | +| `-filelist-url` | string | `https://metadata.ftp-master.debian.org/changelogs/filelist.yaml.xz` | URL of the Debian filelist YAML archive. | +| `-url-base` | string | `https://metadata.ftp-master.debian.org/changelogs` | Base URL for downloading individual copyright files. | +| `-prefix-filter` | string | `main/` | Archive section prefix to filter (e.g., `main/`). | +| `-min-expected-files` | int | `40000` | Minimum expected number of copyright files; fails if fewer are found. | +| `-use-curl` | bool | `false` | Delegate downloads to `curl --parallel` instead of Go worker pool. | +| `-curl-config-file` | string | `""` | Optional path to output the generated curl configuration file. | +| `-curl-config-only` | bool | `false` | Generate the curl configuration file and exit immediately without downloading. | + + +--- + +## Docker & Deployment + +### Building the Docker Container + +Build the container image from the `vulnfeeds/` directory: + +```bash +docker build -t gcr.io/oss-vdb/debian-copyright-mirror:latest -f cmd/mirrors/debian-copyright-mirror/Dockerfile . +``` + +### Kubernetes CronJob Deployment + +In production, this mirror runs as a scheduled Kubernetes `CronJob` in GKE (defined in `deployment/clouddeploy/gke-workers/base/feeds/debian-copyright-mirror.yaml` and environment overlays): + +- **Schedule**: Runs daily (`0 5 * * *` Sydney time). +- **Entrypoint**: Runs the `debian-copyright-mirror` Go binary directly, which reads `WORK_DIR` (e.g. `/scratch`) and `GCS_PATH` (e.g. `gs://cve-osv-conversion/debian_copyright/debian_copyright.tar`) from the environment and streams the archive directly to Cloud Storage. diff --git a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/build.sh b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/build.sh index 1fae76fdb88..375eb4cdaf3 100755 --- a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/build.sh +++ b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/build.sh @@ -16,7 +16,9 @@ set -ex # See the License for the specific language governing permissions and # limitations under the License. +cd ../../.. + docker build \ -t gcr.io/oss-vdb/debian-copyright-mirror:latest \ - -f Dockerfile --pull . && \ + -f cmd/mirrors/debian-copyright-mirror/Dockerfile --pull . && \ gcloud docker -- push gcr.io/oss-vdb/debian-copyright-mirror:latest diff --git a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/debian-copyright-mirror.py b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/debian-copyright-mirror.py deleted file mode 100644 index c4bdfe2d202..00000000000 --- a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/debian-copyright-mirror.py +++ /dev/null @@ -1,163 +0,0 @@ -""" -Download all of the copyright files for packages in main in Debian unstable. - -This: - -Parses https://metadata.ftp-master.debian.org/changelogs/filelist.yaml.xz - to identify the files to retrieve -Generates a curl configuration to download the URLs -Executes curl to download the URLs in parallel -""" - -import os -import argparse -import yaml -import lzma -import urllib.request -import subprocess -from typing import List, Optional, Dict - - -class Error(Exception): - """General purpose error class.""" - - -def download_url_to_directory(url: str, - directory: str, - filename: Optional[str] = None) -> Optional[str]: - """ - Downloads a URL to a specified directory. - - Args: - url (str): The URL to download. - directory (str): The directory to save the file to. - filename (str, optional): The filename to use. If None, the filename - is extracted from the URL. Defaults to None. - - Returns: - str: The full path to the downloaded file, or None on error. - """ - try: - if not os.path.exists(directory): - os.makedirs(directory) - - if filename is None: - filename = os.path.basename(urllib.parse.urlsplit(url).path) - - filepath = os.path.join(directory, filename) - - urllib.request.urlretrieve(url, filepath) - return filepath - - except urllib.error.URLError as e: - print(f'Error downloading {url}: {e}') - return None - except OSError as e: # Catch directory creation/file writing errors. - print(f'OS Error: {e}') - return None - except Exception as e: # Catch any other unexpected error. - print(f'An unexpected error occurred: {e}') - return None - - -def extract_unstable_copyright(filelist: str) -> Dict: - """ - Extracts the 'unstable_copyright' entry for each package - from an xz-compressed YAML file. - - Args: - filelist (str): The path to the xz-compressed YAML filelist - - Returns: - A dictionary where keys are package names and values are - their 'unstable_copyright' entries, or None if no unstable - copyright is found. - """ - try: - with lzma.open(filelist, 'rt', encoding='utf-8') as f: - data = yaml.safe_load(f) - - results = {} - for package, versions in data.items(): - if 'unstable' in versions: - entries = versions['unstable'] - for entry in entries: - if entry.endswith('unstable_copyright'): - results[package] = entry - break # Found it, no need to continue checking this package. - - return results - - except FileNotFoundError: - print(f"Error: File not found at {filelist}") - return None - except lzma.LZMAError as e: - print(f"Error: LZMA decompression failed: {e}") - return None - except yaml.YAMLError as e: - print(f"Error: YAML parsing failed: {e}") - return None - except Exception as e: - print(f"An unexpected error occurred: {e}") - return None - - -def generate_curl_configuration(filelist: List[str]): - """ - Generates a curl configuration to download all of the files in filelist. - - --output filename - url = https://url - - Args: - filelist (List[str]): a list of files to download. - """ - - url_base = 'https://metadata.ftp-master.debian.org/changelogs' - - with open('/tmp/curl_configuration', 'w') as curl_config: - curl_config.writelines([ - '--output ' + path + '\n' + 'url = ' + os.path.join(url_base, path) + - '\n' for path in filelist - ]) - - -def execute_curl(configuration: str, directory: str): - """ - Execute curl with the supplied configuration in the specified directory. - - Args: - configuration (str): path to configuration file. - directory (str): path to set current working directory to. - """ - - os.makedirs(directory) - subprocess.run( - ['curl', '--parallel', '--create-dirs', '--config', configuration], - cwd=directory, - check=True) - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument('work_dir') - args = parser.parse_args() - - download_url_to_directory( - 'https://metadata.ftp-master.debian.org/changelogs/filelist.yaml.xz', - '/tmp') - unstable_package_copyright_files = extract_unstable_copyright( - '/tmp/filelist.yaml.xz') - if unstable_package_copyright_files is None: - raise Error('Unexpected result determining files to download') - generate_curl_configuration( - f for f in unstable_package_copyright_files.values() - if f.startswith('main/')) - with open("/tmp/curl_configuration") as curl_configuration: - if len(curl_configuration.readlines()) < 80000: - raise Error('Unexpectly small curl configuration') - execute_curl('/tmp/curl_configuration', args.work_dir) - - -if __name__ == '__main__': - main() diff --git a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/debian-copyright-mirror.sh b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/debian-copyright-mirror.sh deleted file mode 100755 index a51fdfa08f6..00000000000 --- a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/debian-copyright-mirror.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash -# -# Copyright 2021 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# -# Maintain a GCS bucket mirror of Debian's copyright files -# -# -# -# Inputs: -# * A local work directory -# * GCS bucket name + path to tarball -# - -# Setting BE_VERBOSE to an empty string or null value suppresses silencing of -# commands - -mkdir -p "${WORK_DIR}" || true - -python3 debian-copyright-mirror.py "${WORK_DIR}/metadata.ftp-master.debian.org/changelogs/" - -tar -C "${WORK_DIR}" -cf "${WORK_DIR}/$(basename ${GCS_PATH})" . - -gcloud storage ${BE_VERBOSE="--quiet"} cp "${WORK_DIR}/$(basename ${GCS_PATH})" "${GCS_PATH}" diff --git a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main.go b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main.go new file mode 100644 index 00000000000..09782dcb556 --- /dev/null +++ b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main.go @@ -0,0 +1,269 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package main downloads all machine-readable copyright files for packages in Debian unstable. +package main + +import ( + "bufio" + "context" + "flag" + "fmt" + "io" + "log/slog" + "net/http" + "os" + "os/exec" + "path/filepath" + "strings" + + "cloud.google.com/go/storage" + gcs "github.com/google/osv.dev/vulnfeeds/gcs-tools" + "github.com/google/osv.dev/vulnfeeds/utility/logger" +) + +const ( + DefaultFilelistURL = "https://metadata.ftp-master.debian.org/changelogs/filelist.yaml.xz" + DefaultURLBase = "https://metadata.ftp-master.debian.org/changelogs" + DefaultPrefixFilter = "main/" + DefaultMinExpectedFiles = 40000 +) + +// ExtractUnstableCopyright parses Debian changelogs filelist YAML content from a reader +// and extracts the unstable_copyright path for each package matching the specified prefixFilter. +func ExtractUnstableCopyright(r io.Reader, prefixFilter string) ([]string, error) { + scanner := bufio.NewScanner(r) + const maxLineLen = 1024 * 1024 + buf := make([]byte, 64*1024) + scanner.Buffer(buf, maxLineLen) + + var results []string + inUnstable := false + currentPkgFound := false + + for scanner.Scan() { + line := scanner.Text() + if len(line) > 0 && line[0] != ' ' && line[0] != '\t' && strings.HasSuffix(line, ":") { + inUnstable = false + currentPkgFound = false + + continue + } + + trimmed := strings.TrimSpace(line) + if trimmed == "" || strings.HasPrefix(trimmed, "#") { + continue + } + + if strings.HasPrefix(line, " ") && !strings.HasPrefix(line, " -") && strings.HasSuffix(line, ":") { + inUnstable = trimmed == "unstable:" || trimmed == "'unstable':" || trimmed == `"unstable":` + + continue + } + + if inUnstable && !currentPkgFound && strings.HasPrefix(trimmed, "- ") { + entry := strings.Trim(strings.TrimSpace(strings.TrimPrefix(trimmed, "- ")), `"'`) + if strings.HasSuffix(entry, "unstable_copyright") && (prefixFilter == "" || strings.HasPrefix(entry, prefixFilter)) { + results = append(results, entry) + currentPkgFound = true + } + } + } + + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("error scanning filelist YAML: %w", err) + } + + return results, nil +} + +func fetchCopyrightFiles(ctx context.Context, filelistURL, prefixFilter string) ([]string, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, filelistURL, nil) + if err != nil { + return nil, err + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("HTTP %d fetching %s", resp.StatusCode, filelistURL) + } + + cmd := exec.CommandContext(ctx, "xz", "-dc") + cmd.Stdin = resp.Body + cmd.Stderr = os.Stderr + stdout, err := cmd.StdoutPipe() + if err != nil { + return nil, err + } + if err := cmd.Start(); err != nil { + return nil, err + } + + files, err := ExtractUnstableCopyright(stdout, prefixFilter) + _ = cmd.Wait() + + return files, err +} + +// GenerateCurlConfiguration generates a curl config file for parallel downloads. +func GenerateCurlConfiguration(filelist []string, urlBase, configPath string) error { + file, err := os.Create(configPath) + if err != nil { + return fmt.Errorf("failed to create curl config file: %w", err) + } + defer file.Close() + + w := bufio.NewWriter(file) + for _, path := range filelist { + fullURL := fmt.Sprintf("%s/%s", strings.TrimRight(urlBase, "/"), strings.TrimLeft(path, "/")) + if _, err := fmt.Fprintf(w, "--output %s\nurl = %s\n", path, fullURL); err != nil { + return fmt.Errorf("failed to write curl config entry: %w", err) + } + } + + return w.Flush() +} + +// ExecuteCurl runs curl with the specified configuration file in the working directory. +func ExecuteCurl(ctx context.Context, configPath, workDir string) error { + if err := os.MkdirAll(workDir, 0755); err != nil { + return fmt.Errorf("failed to create work directory: %w", err) + } + + cmd := exec.CommandContext(ctx, "curl", "--parallel", "--create-dirs", "--config", configPath) + cmd.Dir = workDir + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + logger.Info("Executing curl in parallel", slog.String("config", configPath), slog.String("workDir", workDir)) + if err := cmd.Run(); err != nil { + return fmt.Errorf("curl execution failed: %w", err) + } + + return nil +} + +// CreateTarArchive creates a tar archive of workDir using the system tar command. +func CreateTarArchive(ctx context.Context, workDir, tarPath string) error { + // #nosec G204 -- arguments are locally configured paths + cmd := exec.CommandContext(ctx, "tar", "-C", workDir, "--exclude="+filepath.Base(tarPath), "-cf", tarPath, ".") + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + logger.Info("Creating tar archive", slog.String("workDir", workDir), slog.String("tarPath", tarPath)) + + return cmd.Run() +} + +func uploadToGCS(ctx context.Context, tarPath, gcsURI string) error { + bucketName, objectName, err := gcs.ParseGCSPath(gcsURI) + if err != nil { + return err + } + client, err := storage.NewClient(ctx) + if err != nil { + return fmt.Errorf("failed to create GCS client: %w", err) + } + defer client.Close() + + logger.Info("Uploading tar archive to GCS", slog.String("bucket", bucketName), slog.String("object", objectName)) + + return gcs.UploadFile(ctx, client.Bucket(bucketName), objectName, tarPath) +} + +func main() { + logger.InitGlobalLogger() + defer logger.Close() + + filelistURL := flag.String("filelist-url", DefaultFilelistURL, "URL of the Debian filelist.yaml.xz file.") + urlBase := flag.String("url-base", DefaultURLBase, "Base URL for downloading Debian changelog/copyright files.") + prefixFilter := flag.String("prefix-filter", DefaultPrefixFilter, "Prefix filter for package paths to download (e.g. 'main/').") + minExpectedFiles := flag.Int("min-expected-files", DefaultMinExpectedFiles, "Minimum expected number of copyright files.") + outDirFlag := flag.String("out-dir", "", "Directory to download copyright files into (defaults to WORK_DIR env var or ./debian_copyright).") + workDirFlag := flag.String("work-dir", "", "Alias for -out-dir.") + tarPathFlag := flag.String("tar-path", "", "Optional local destination path for tarball archive.") + gcsPathFlag := flag.String("gcs-path", "", "Destination GCS path for tarball archive (defaults to GCS_PATH env var).") + + flag.Parse() + + workDir := *outDirFlag + if *workDirFlag != "" { + workDir = *workDirFlag + } else if flag.NArg() > 0 { + workDir = flag.Arg(0) + } else if workDir == "" { + if envWorkDir := os.Getenv("WORK_DIR"); envWorkDir != "" { + workDir = envWorkDir + } else { + workDir = "debian_copyright" + } + } + workDir = filepath.Clean(workDir) + + ctx := context.Background() + + logger.Info("Fetching and extracting copyright filelist", slog.String("url", *filelistURL)) + copyrightFiles, err := fetchCopyrightFiles(ctx, *filelistURL, *prefixFilter) + if err != nil { + logger.Fatal("Failed to obtain copyright file list", slog.Any("err", err)) + } + + logger.Info("Discovered copyright files", slog.Int("count", len(copyrightFiles))) + if len(copyrightFiles) < *minExpectedFiles { + logger.Fatal("Unexpectedly small number of copyright files found", + slog.Int("found", len(copyrightFiles)), + slog.Int("min_expected", *minExpectedFiles), + ) + } + + tempDir, err := os.MkdirTemp("", "debian-copyright-mirror-*") + if err != nil { + logger.Fatal("Failed to create temp dir", slog.Any("err", err)) + } + defer os.RemoveAll(tempDir) + + cfgPath := filepath.Join(tempDir, "curl_configuration") + if err := GenerateCurlConfiguration(copyrightFiles, *urlBase, cfgPath); err != nil { + logger.Fatal("Failed to generate curl configuration", slog.Any("err", err)) + } + + if err := ExecuteCurl(ctx, cfgPath, workDir); err != nil { + logger.Fatal("Curl download failed", slog.Any("err", err)) + } + + gcsDest := *gcsPathFlag + if gcsDest == "" { + gcsDest = os.Getenv("GCS_PATH") + } + + tarDest := *tarPathFlag + if tarDest == "" && gcsDest != "" { + tarDest = filepath.Join(workDir, filepath.Base(gcsDest)) + } + + if tarDest != "" { + if err := CreateTarArchive(ctx, workDir, tarDest); err != nil { + logger.Fatal("Failed to create tar archive", slog.Any("err", err)) + } + if gcsDest != "" { + if err := uploadToGCS(ctx, tarDest, gcsDest); err != nil { + logger.Fatal("Failed to upload tar archive to GCS", slog.Any("err", err)) + } + } + } + + logger.Info("Debian copyright mirror sync completed successfully.", slog.String("workDir", workDir)) +} diff --git a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main_test.go b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main_test.go new file mode 100644 index 00000000000..f772ffff114 --- /dev/null +++ b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main_test.go @@ -0,0 +1,213 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "reflect" + "strings" + "testing" +) + +const sampleFilelistYAML = ` +0ad: + 0.0.17-1: + - main/0/0ad/0ad_0.0.17-1_copyright + - main/0/0ad/0ad_0.0.17-1_changelog + testing: + - main/0/0ad/testing_copyright + - main/0/0ad/testing_changelog + unstable: + - main/0/0ad/unstable_changelog + - main/0/0ad/unstable_copyright + - main/0/0ad/unstable_NEWS +0ad-data: + unstable: + - main/0/0ad-data/unstable_changelog + - main/0/0ad-data/unstable_copyright +non-free-pkg: + unstable: + - non-free/n/non-free-pkg/unstable_changelog + - non-free/n/non-free-pkg/unstable_copyright +no-unstable-pkg: + stable: + - main/n/no-unstable-pkg/stable_copyright +no-copyright-pkg: + unstable: + - main/n/no-copyright-pkg/unstable_changelog +'0xffff': + 0.6.1-1: + - main/0/0xffff/0xffff_0.6.1-1_changelog + unstable: + - "main/0/0xffff/unstable_changelog" + - 'main/0/0xffff/unstable_copyright' +` + +func TestExtractUnstableCopyright(t *testing.T) { + tests := []struct { + name string + yamlContent string + prefixFilter string + expected []string + }{ + { + name: "Sample filelist with main/ filter", + yamlContent: sampleFilelistYAML, + prefixFilter: "main/", + expected: []string{ + "main/0/0ad/unstable_copyright", + "main/0/0ad-data/unstable_copyright", + "main/0/0xffff/unstable_copyright", + }, + }, + { + name: "Sample filelist without filter", + yamlContent: sampleFilelistYAML, + prefixFilter: "", + expected: []string{ + "main/0/0ad/unstable_copyright", + "main/0/0ad-data/unstable_copyright", + "non-free/n/non-free-pkg/unstable_copyright", + "main/0/0xffff/unstable_copyright", + }, + }, + { + name: "Empty YAML", + yamlContent: "", + prefixFilter: "main/", + expected: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := strings.NewReader(tt.yamlContent) + got, err := ExtractUnstableCopyright(r, tt.prefixFilter) + if err != nil { + t.Fatalf("ExtractUnstableCopyright unexpected error: %v", err) + } + if !reflect.DeepEqual(got, tt.expected) { + t.Errorf("ExtractUnstableCopyright() = %v, want %v", got, tt.expected) + } + }) + } +} + +func TestGenerateCurlConfiguration(t *testing.T) { + tempDir := t.TempDir() + configPath := filepath.Join(tempDir, "curl_config") + + files := []string{ + "main/0/0ad/unstable_copyright", + "main/0/0xffff/unstable_copyright", + } + urlBase := "https://metadata.ftp-master.debian.org/changelogs" + + if err := GenerateCurlConfiguration(files, urlBase, configPath); err != nil { + t.Fatalf("GenerateCurlConfiguration returned error: %v", err) + } + + content, err := os.ReadFile(configPath) + if err != nil { + t.Fatalf("Failed to read generated config: %v", err) + } + + expected := "--output main/0/0ad/unstable_copyright\n" + + "url = https://metadata.ftp-master.debian.org/changelogs/main/0/0ad/unstable_copyright\n" + + "--output main/0/0xffff/unstable_copyright\n" + + "url = https://metadata.ftp-master.debian.org/changelogs/main/0/0xffff/unstable_copyright\n" + + if string(content) != expected { + t.Errorf("GenerateCurlConfiguration() =\n%s\nwant:\n%s", string(content), expected) + } +} + +func TestExecuteCurl(t *testing.T) { + fileMap := map[string]string{ + "main/a/pkg1/unstable_copyright": "Copyright pkg1\n", + "main/b/pkg2/unstable_copyright": "Copyright pkg2\n", + } + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + path := strings.TrimPrefix(r.URL.Path, "/") + if content, ok := fileMap[path]; ok { + w.WriteHeader(http.StatusOK) + _, _ = fmt.Fprint(w, content) + + return + } + w.WriteHeader(http.StatusNotFound) + })) + defer ts.Close() + + tempDir := t.TempDir() + configPath := filepath.Join(tempDir, "curl_config") + workDir := filepath.Join(tempDir, "work") + + filelist := []string{ + "main/a/pkg1/unstable_copyright", + "main/b/pkg2/unstable_copyright", + } + + if err := GenerateCurlConfiguration(filelist, ts.URL, configPath); err != nil { + t.Fatalf("GenerateCurlConfiguration failed: %v", err) + } + + if err := ExecuteCurl(context.Background(), configPath, workDir); err != nil { + t.Fatalf("ExecuteCurl failed: %v", err) + } + + for _, relPath := range filelist { + data, err := os.ReadFile(filepath.Join(workDir, relPath)) + if err != nil { + t.Errorf("Failed to read downloaded file %s: %v", relPath, err) + continue + } + if string(data) != fileMap[relPath] { + t.Errorf("Content mismatch for %s: got %q, want %q", relPath, string(data), fileMap[relPath]) + } + } +} + +func TestCreateTarArchive(t *testing.T) { + tempDir := t.TempDir() + workDir := filepath.Join(tempDir, "work") + subDir := filepath.Join(workDir, "main", "a", "pkg1") + if err := os.MkdirAll(subDir, 0755); err != nil { + t.Fatalf("Failed to create test directory: %v", err) + } + filePath := filepath.Join(subDir, "unstable_copyright") + if err := os.WriteFile(filePath, []byte("test copyright content"), 0600); err != nil { + t.Fatalf("Failed to write test file: %v", err) + } + + tarPath := filepath.Join(workDir, "test_archive.tar") + if err := CreateTarArchive(context.Background(), workDir, tarPath); err != nil { + t.Fatalf("CreateTarArchive failed: %v", err) + } + + fi, err := os.Stat(tarPath) + if err != nil { + t.Fatalf("Expected tar file to exist: %v", err) + } + if fi.Size() == 0 { + t.Errorf("Expected non-empty tar archive") + } +} diff --git a/vulnfeeds/gcs-tools/gcs.go b/vulnfeeds/gcs-tools/gcs.go index 8fc78d7b4ff..7ccdd37a3a7 100644 --- a/vulnfeeds/gcs-tools/gcs.go +++ b/vulnfeeds/gcs-tools/gcs.go @@ -145,6 +145,20 @@ func UploadToGCS(ctx context.Context, bkt *storage.BucketHandle, objectName stri return nil } +// ParseGCSPath parses a gs:// URI into bucket and object name components. +func ParseGCSPath(gcsURI string) (bucket, object string, err error) { + if !strings.HasPrefix(gcsURI, "gs://") { + return "", "", fmt.Errorf("invalid GCS URI: %s (must start with gs://)", gcsURI) + } + trimmed := strings.TrimPrefix(gcsURI, "gs://") + parts := strings.SplitN(trimmed, "/", 2) + if len(parts) < 2 || parts[0] == "" || parts[1] == "" { + return "", "", fmt.Errorf("invalid GCS URI format %q (expected gs://bucket/object)", gcsURI) + } + + return parts[0], parts[1], nil +} + // UploadFile uploads a local file to a GCS bucket. func UploadFile(ctx context.Context, bkt *storage.BucketHandle, objectName string, filePath string) error { f, err := os.Open(filePath) diff --git a/vulnfeeds/gcs-tools/gcs_test.go b/vulnfeeds/gcs-tools/gcs_test.go index 65d0514ad77..01cdf5deb61 100644 --- a/vulnfeeds/gcs-tools/gcs_test.go +++ b/vulnfeeds/gcs-tools/gcs_test.go @@ -299,3 +299,50 @@ func TestListObjectsFast(t *testing.T) { t.Errorf("ListObjectsFast returned unexpected list.\ngot: %v\nwant: %v", got, expected) } } + +func TestParseGCSPath(t *testing.T) { + tests := []struct { + input string + wantBucket string + wantObject string + expectError bool + }{ + { + input: "gs://my-bucket/path/to/file.tar", + wantBucket: "my-bucket", + wantObject: "path/to/file.tar", + expectError: false, + }, + { + input: "gs://cve-osv-conversion/debian_copyright.tar", + wantBucket: "cve-osv-conversion", + wantObject: "debian_copyright.tar", + expectError: false, + }, + { + input: "/local/path/file.tar", + expectError: true, + }, + { + input: "gs://bucket-only", + expectError: true, + }, + { + input: "gs://", + expectError: true, + }, + } + + for _, tt := range tests { + bucket, obj, err := ParseGCSPath(tt.input) + if (err != nil) != tt.expectError { + t.Errorf("ParseGCSPath(%q) error = %v, expectError = %v", tt.input, err, tt.expectError) + continue + } + if !tt.expectError { + if bucket != tt.wantBucket || obj != tt.wantObject { + t.Errorf("ParseGCSPath(%q) = (%q, %q), want (%q, %q)", tt.input, bucket, obj, tt.wantBucket, tt.wantObject) + } + } + } +}