diff --git a/README.md b/README.md index ede54ed..c699a4d 100644 --- a/README.md +++ b/README.md @@ -117,6 +117,47 @@ ratt -recheck -exclude '^(gcc-9|gcc-8|llvm-toolchain-10|libreoffice|trilinos|llv **Note**: you need to escape the `+` sign in package names as in `dbus-c\+\+` to avoid messing up the regexp expression. +# Transition-aware candidate selection + +For library transitions, using all binary packages from the `.changes` file as +reverse build-dependency roots can be too broad. The `-transition_affected` +option instead scans the selected `Packages` indexes, finds binary packages +whose parsed `Depends` package names match the supplied regex, maps those +binaries back to source packages, and rebuilds those source packages with the +`.deb`s from the `.changes` file still injected via `sbuild --extra-package`. +The argument is only a regex matched against parsed dependency package names, +it is not a full Ben expression such as `.depends ~ /.../`, and it is not +matched against the raw `Depends` field text. In practice, use anchored package +name regexes such as `^(libfoo2|libfoo1)$`. + +For instance: + +``` +ratt -transition_affected '^(libpoppler\-cpp3|libpoppler156|libpoppler\-cpp2|libpoppler147)$' ../poppler_*.changes +``` + +## Choosing the regex + +If a Ben transition tracker exists, start from its `Affected` expression. For +instance, convert `Affected: .depends ~ /\b(libfoo2|libfoo1)\b/` to +`-transition_affected '^(libfoo2|libfoo1)$'`. + +If there is no tracker yet, build the regex from the old and new runtime +library package names for the transition. Include both, since some packages may +still depend on the old package while others already depend on the new one. Do +not copy every binary from the `.changes` file into the regex. Leave out +unrelated packages such as `-dev`, `-doc`, dbgsym, or utilities, unless they +are actually part of the transition. + +:warning: `-direct-rdeps` and `-rdeps-depth` are ignored in this mode because +selection is based on binary package `Depends`, not on `dose-ceve` reverse +dependency traversal. + +If a matching binary maps to a source package that is not present in the +selected `Sources` indexes, ratt logs a warning and cannot schedule that source +for rebuild. When comparing with Ben transition pages, make sure your local +archive metadata includes the same relevant components, for example `contrib` +when transition consumers live there. # Using `-chdist` to target multiple Debian suites diff --git a/docs/ratt.rst b/docs/ratt.rst index 3e2a6b2..d831523 100644 --- a/docs/ratt.rst +++ b/docs/ratt.rst @@ -20,6 +20,7 @@ SYNOPSIS [-dist DIST] [-sbuild_dist DIST] [-sbuild-experimental-aspcud] [-sbuild-keep-build-log] [-log_dir DIR] [-chdist NAME] [-direct-rdeps] [-rdeps-depth N] + [-transition_affected REGEX] [-json] .changes DESCRIPTION @@ -86,6 +87,25 @@ OPTIONS included.  See the ``--depth`` option in ``dose-ceve(1)`` manpage to see more details. +**-transition_affected** *regex* + Select source packages for a transition by scanning parsed binary package + ``Depends`` from the selected ``Packages`` indexes. If a parsed dependency + package name matches the regex, the binary package is mapped back to its + source package, and ratt rebuilds that source package while still injecting + the ``.deb`` files from the required ``.changes`` file. + + This mode does not use the binaries from the ``.changes`` file as reverse + build-dependency roots. The argument is only a regex matching parsed + dependency package names, not a regex matched against the full raw ``Depends`` + field. Users should usually pass anchored package name regexes such as + ``^(libfoo2|libfoo1)$``. + + If a matching binary maps to a source package that is not present in the + selected ``Sources`` indexes, ratt logs a warning and cannot schedule that + source for rebuild. When comparing with Ben transition pages, ensure the local + archive metadata includes the same relevant components, such as ``contrib`` + for transition consumers that live there. + **-json** Output results in JSON format (currently only works in combination with `-dry_run`). JSON is written to stdout; human-readable logs go to stderr. Each @@ -143,6 +163,23 @@ Limit to direct reverse build-dependencies only:: $ ratt -direct-rdeps yourpackage_*.changes +Transition-aware candidate selection:: + + $ ratt -transition_affected '^(libfoo2|libfoo1)$' yourpackage_*.changes + +Choosing the transition regex: + +If a Ben transition tracker exists, start from its ``Affected`` expression. For +example, convert ``Affected: .depends ~ /\b(libfoo2|libfoo1)\b/`` to +``-transition_affected '^(libfoo2|libfoo1)$'``. + +If there is no tracker yet, build the regex from the old and new runtime +library package names for the transition. Include both, since some packages may +still depend on the old package while others already depend on the new one. Do +not copy every binary from the ``.changes`` file into the regex. Leave out +unrelated packages such as ``-dev``, ``-doc``, dbgsym, or utilities, unless +they are actually part of the transition. + Print dry-run result in JSON format:: $ ratt -dry_run -json yourpackage_*.changes diff --git a/ratt.go b/ratt.go index c9efced..5394f52 100644 --- a/ratt.go +++ b/ratt.go @@ -104,6 +104,10 @@ var ( 0, "Set the maximum depth for reverse dependency resolution. For more details, see the --depth option in the dose-ceve(1) manpage") + transitionAffected = flag.String("transition_affected", + "", + "Select transition rebuild candidates by matching binary Depends package names against a regex") + jsonOutput = flag.Bool("json", false, "Output results in JSON format (currently only works in combination with -dry_run)") @@ -225,24 +229,31 @@ func dependsOn(src control.SourceIndex, binaries map[string]bool) bool { return false } -func addReverseBuildDeps(sourcesPath string, binaries map[string]bool, rebuild map[string][]version.Version) error { - log.Printf("Loading sources index %q\n", sourcesPath) +func aptIndexReader(indexPath string) (*bufio.Reader, func(), error) { catFile := exec.Command("/usr/lib/apt/apt-helper", "cat-file", - sourcesPath) - var s *bufio.Reader + indexPath) if lines, err := catFile.Output(); err == nil { - s = bufio.NewReader(bytes.NewReader(lines)) - } else { - // Fallback for older versions of apt-get. See - // <20160111171230.GA17291@debian.org> for context. - o, err := os.Open(sourcesPath) - if err != nil { - return err - } - defer o.Close() - s = bufio.NewReader(o) + return bufio.NewReader(bytes.NewReader(lines)), func() {}, nil } + + // Fallback for older versions of apt-get. See + // <20160111171230.GA17291@debian.org> for context. + o, err := os.Open(indexPath) + if err != nil { + return nil, nil, err + } + return bufio.NewReader(o), func() { o.Close() }, nil +} + +func addReverseBuildDeps(sourcesPath string, binaries map[string]bool, rebuild map[string][]version.Version) error { + log.Printf("Loading sources index %q\n", sourcesPath) + s, cleanup, err := aptIndexReader(sourcesPath) + if err != nil { + return err + } + defer cleanup() + idx, err := control.ParseSourceIndex(s) if err != nil && err != io.EOF { return err @@ -272,6 +283,91 @@ func fallback(sourcesPaths []string, binaries []string) (map[string][]version.Ve return rebuild, nil } +func binaryDependsMatchesAffected(bin control.BinaryIndex, affected *regexp.Regexp) bool { + depends := bin.GetDepends() + for _, possibility := range depends.GetAllPossibilities() { + if affected.MatchString(possibility.Name) { + return true + } + } + return false +} + +func addTransitionAffectedSources(packagesPath string, affected *regexp.Regexp, selected map[string]struct{}) error { + log.Printf("Loading packages index %q\n", packagesPath) + p, cleanup, err := aptIndexReader(packagesPath) + if err != nil { + return err + } + defer cleanup() + + idx, err := control.ParseBinaryIndex(p) + if err != nil && err != io.EOF { + return err + } + + for _, bin := range idx { + if !binaryDependsMatchesAffected(bin, affected) { + continue + } + selected[bin.SourcePackage()] = struct{}{} + } + + return nil +} + +func addSelectedSourceVersions(sourcesPath string, selected map[string]struct{}, rebuild map[string][]version.Version) error { + log.Printf("Loading sources index %q\n", sourcesPath) + s, cleanup, err := aptIndexReader(sourcesPath) + if err != nil { + return err + } + defer cleanup() + + idx, err := control.ParseSourceIndex(s) + if err != nil && err != io.EOF { + return err + } + + for _, src := range idx { + if _, ok := selected[src.Package]; ok { + rebuild[src.Package] = append(rebuild[src.Package], src.Version) + } + } + + return nil +} + +func transitionAffectedSources(packagesPaths, sourcesPaths []string, affectedRegex string) (map[string][]version.Version, error) { + affected, err := regexp.Compile(affectedRegex) + if err != nil { + return nil, err + } + + selected := make(map[string]struct{}) + for _, packagesPath := range packagesPaths { + if err := addTransitionAffectedSources(packagesPath, affected, selected); err != nil { + return nil, err + } + } + log.Printf("Found %d source packages with binary Depends matching -transition_affected\n", len(selected)) + + rebuild := make(map[string][]version.Version) + for _, sourcesPath := range sourcesPaths { + if err := addSelectedSourceVersions(sourcesPath, selected, rebuild); err != nil { + return nil, err + } + } + + for src := range selected { + if _, ok := rebuild[src]; !ok { + log.Printf("Warning: source package %q selected by -transition_affected was not found in any Sources index", src) + } + } + + return rebuild, nil +} + func resolveAptListFile(indexFilePath string) (string, error) { cmd := exec.Command("/usr/lib/apt/apt-helper", "cat-file", indexFilePath) resolvedContent, err := cmd.Output() @@ -636,7 +732,20 @@ func main() { } } - rebuild, err := reverseBuildDeps(packagesPaths, sourcesPaths, binaries) + var rebuild map[string][]version.Version + var err error + if *transitionAffected != "" { + if *directRdeps { + log.Printf("Warning: --direct-rdeps is ignored in -transition_affected mode") + } + if *rdepsDepth != 0 { + log.Printf("Warning: --rdeps-depth=%d is ignored in -transition_affected mode", *rdepsDepth) + } + log.Printf("Selecting rebuild candidates using -transition_affected=%q\n", *transitionAffected) + rebuild, err = transitionAffectedSources(packagesPaths, sourcesPaths, *transitionAffected) + } else { + rebuild, err = reverseBuildDeps(packagesPaths, sourcesPaths, binaries) + } if err != nil { log.Fatal(err) } diff --git a/ratt_test.go b/ratt_test.go new file mode 100644 index 0000000..531ae82 --- /dev/null +++ b/ratt_test.go @@ -0,0 +1,221 @@ +package main + +import ( + "bytes" + "log" + "os" + "path/filepath" + "sort" + "strings" + "testing" + + "pault.ag/go/debian/version" +) + +func writeIndex(t *testing.T, dir, name, contents string) string { + t.Helper() + + path := filepath.Join(dir, name) + if err := os.WriteFile(path, []byte(contents), 0644); err != nil { + t.Fatal(err) + } + return path +} + +func sortedSourceNames(rebuild map[string][]version.Version) []string { + names := make([]string, 0, len(rebuild)) + for name := range rebuild { + names = append(names, name) + } + sort.Strings(names) + return names +} + +func TestTransitionAffectedSourcesSelectsSourceVersionsAndDeduplicates(t *testing.T) { + dir := t.TempDir() + packagesPath := writeIndex(t, dir, "Packages", `Package: foo-bin +Source: foo +Version: 1.0-1+b1 +Architecture: amd64 +Depends: libaffected1 (>= 1), libc6 + +Package: foo-tools +Source: foo +Version: 1.0-1+b1 +Architecture: amd64 +Depends: libaffected1, libc6 + +Package: bar-bin +Source: bar +Version: 2.0-1 +Architecture: amd64 +Depends: libunrelated1 + +`) + sourcesPath := writeIndex(t, dir, "Sources", `Package: foo +Binary: foo-bin, foo-tools +Version: 1.0-1 + +Package: bar +Binary: bar-bin +Version: 2.0-1 + +`) + + rebuild, err := transitionAffectedSources([]string{packagesPath}, []string{sourcesPath}, `^libaffected1$`) + if err != nil { + t.Fatal(err) + } + + if got, want := sortedSourceNames(rebuild), []string{"foo"}; len(got) != len(want) || got[0] != want[0] { + t.Fatalf("selected sources = %v, want %v", got, want) + } + if got, want := rebuild["foo"][0].String(), "1.0-1"; got != want { + t.Fatalf("foo version = %q, want %q", got, want) + } +} + +func TestTransitionAffectedSourcesMapsBinaryToSourcePackage(t *testing.T) { + dir := t.TempDir() + packagesPath := writeIndex(t, dir, "Packages", `Package: self-src +Version: 1.0-1 +Architecture: amd64 +Depends: libaffected1 + +Package: nmu-bin +Source: nmu-src (2.0-1) +Version: 2.0-1+b1 +Architecture: amd64 +Depends: libaffected1 + +`) + sourcesPath := writeIndex(t, dir, "Sources", `Package: self-src +Binary: self-src +Version: 1.0-1 + +Package: nmu-src +Binary: nmu-bin +Version: 2.0-1 + +`) + + rebuild, err := transitionAffectedSources([]string{packagesPath}, []string{sourcesPath}, `^libaffected1$`) + if err != nil { + t.Fatal(err) + } + + if got, want := sortedSourceNames(rebuild), []string{"nmu-src", "self-src"}; len(got) != len(want) || got[0] != want[0] || got[1] != want[1] { + t.Fatalf("selected sources = %v, want %v", got, want) + } +} + +func TestTransitionAffectedSourcesKeepsVersionsFromMultipleSourceIndexes(t *testing.T) { + dir := t.TempDir() + packagesPath := writeIndex(t, dir, "Packages", `Package: foo-bin +Source: foo +Version: 1.0-1 +Architecture: amd64 +Depends: libaffected1 + +`) + unstableSourcesPath := writeIndex(t, dir, "Sources.unstable", `Package: foo +Binary: foo-bin +Version: 1.0-1 + +`) + experimentalSourcesPath := writeIndex(t, dir, "Sources.experimental", `Package: foo +Binary: foo-bin +Version: 1.1-1 + +`) + + rebuild, err := transitionAffectedSources( + []string{packagesPath}, + []string{unstableSourcesPath, experimentalSourcesPath}, + `^libaffected1$`, + ) + if err != nil { + t.Fatal(err) + } + + if got, want := len(rebuild["foo"]), 2; got != want { + t.Fatalf("foo versions count = %d, want %d", got, want) + } + if got, want := rebuild["foo"][0].String(), "1.0-1"; got != want { + t.Fatalf("first foo version = %q, want %q", got, want) + } + if got, want := rebuild["foo"][1].String(), "1.1-1"; got != want { + t.Fatalf("second foo version = %q, want %q", got, want) + } +} + +func TestTransitionAffectedSourcesOmitsSelectedSourceMissingFromSources(t *testing.T) { + dir := t.TempDir() + packagesPath := writeIndex(t, dir, "Packages", `Package: missing-bin +Source: missing-src +Version: 1.0-1 +Architecture: amd64 +Depends: libaffected1 + +`) + sourcesPath := writeIndex(t, dir, "Sources", `Package: unrelated-src +Binary: unrelated-bin +Version: 1.0-1 + +`) + + var logs bytes.Buffer + oldLogOutput := log.Writer() + log.SetOutput(&logs) + defer log.SetOutput(oldLogOutput) + + rebuild, err := transitionAffectedSources([]string{packagesPath}, []string{sourcesPath}, `^libaffected1$`) + if err != nil { + t.Fatal(err) + } + if len(rebuild) != 0 { + t.Fatalf("selected sources = %v, want none", sortedSourceNames(rebuild)) + } + if !strings.Contains(logs.String(), `Warning: source package "missing-src" selected by -transition_affected was not found in any Sources index`) { + t.Fatalf("missing source warning not logged; logs:\n%s", logs.String()) + } +} + +func TestTransitionAffectedSourcesIgnoresPackagesWithoutMatchingDepends(t *testing.T) { + dir := t.TempDir() + packagesPath := writeIndex(t, dir, "Packages", `Package: no-depends +Source: no-depends-src +Version: 1.0-1 +Architecture: amd64 + +Package: unrelated +Source: unrelated-src +Version: 1.0-1 +Architecture: amd64 +Depends: libunrelated1 + +`) + sourcesPath := writeIndex(t, dir, "Sources", `Package: no-depends-src +Binary: no-depends +Version: 1.0-1 + +Package: unrelated-src +Binary: unrelated +Version: 1.0-1 + +`) + + rebuild, err := transitionAffectedSources([]string{packagesPath}, []string{sourcesPath}, `^libaffected1$`) + if err != nil { + t.Fatal(err) + } + if len(rebuild) != 0 { + t.Fatalf("selected sources = %v, want none", sortedSourceNames(rebuild)) + } +} + +func TestTransitionAffectedSourcesRejectsInvalidRegex(t *testing.T) { + if _, err := transitionAffectedSources(nil, nil, `[`); err == nil { + t.Fatal("transitionAffectedSources accepted an invalid regex") + } +}