Add transition aware reverse rebuild selection

README.md

@@ -117,6 +117,47 @@ ratt -recheck -exclude '^(gcc-9|gcc-8|llvm-toolchain-10|libreoffice|trilinos|llv
 
 **Note**: you need to escape the `+` sign in package names as in `dbus-c\+\+` to avoid messing up the regexp expression.
 
+# Transition-aware candidate selection
+
+For library transitions, using all binary packages from the `.changes` file as
+reverse build-dependency roots can be too broad. The `-transition_affected`
+option instead scans the selected `Packages` indexes, finds binary packages
+whose parsed `Depends` package names match the supplied regex, maps those
+binaries back to source packages, and rebuilds those source packages with the
+`.deb`s from the `.changes` file still injected via `sbuild --extra-package`.
+The argument is only a regex matched against parsed dependency package names,
+it is not a full Ben expression such as `.depends ~ /.../`, and it is not
+matched against the raw `Depends` field text. In practice, use anchored package
+name regexes such as `^(libfoo2|libfoo1)$`.
+
+For instance:
+
+```
+ratt -transition_affected '^(libpoppler\-cpp3|libpoppler156|libpoppler\-cpp2|libpoppler147)$' ../poppler_*.changes
+```
+
+## Choosing the regex
+
+If a Ben transition tracker exists, start from its `Affected` expression. For
+instance, convert `Affected: .depends ~ /\b(libfoo2|libfoo1)\b/` to
+`-transition_affected '^(libfoo2|libfoo1)$'`.
+
+If there is no tracker yet, build the regex from the old and new runtime
+library package names for the transition. Include both, since some packages may
+still depend on the old package while others already depend on the new one. Do
+not copy every binary from the `.changes` file into the regex. Leave out
+unrelated packages such as `-dev`, `-doc`, dbgsym, or utilities, unless they
+are actually part of the transition.
+
+:warning: `-direct-rdeps` and `-rdeps-depth` are ignored in this mode because
+selection is based on binary package `Depends`, not on `dose-ceve` reverse
+dependency traversal.
+
+If a matching binary maps to a source package that is not present in the
+selected `Sources` indexes, ratt logs a warning and cannot schedule that source
+for rebuild. When comparing with Ben transition pages, make sure your local
+archive metadata includes the same relevant components, for example `contrib`
+when transition consumers live there.
 
 # Using `-chdist` to target multiple Debian suites
 

docs/ratt.rst

@@ -20,6 +20,7 @@ SYNOPSIS
         [-dist DIST] [-sbuild_dist DIST] [-sbuild-experimental-aspcud] [-sbuild-keep-build-log]
         [-log_dir DIR] [-chdist NAME]
         [-direct-rdeps] [-rdeps-depth N]
+        [-transition_affected REGEX]
         [-json] <file>.changes
 
 DESCRIPTION
@@ -86,6 +87,25 @@ OPTIONS
  included.  See the ``--depth`` option in ``dose-ceve(1)`` manpage to see
  more details.
 
+**-transition_affected** *regex*
+ Select source packages for a transition by scanning parsed binary package
+ ``Depends`` from the selected ``Packages`` indexes. If a parsed dependency
+ package name matches the regex, the binary package is mapped back to its
+ source package, and ratt rebuilds that source package while still injecting
+ the ``.deb`` files from the required ``.changes`` file.
+
+ This mode does not use the binaries from the ``.changes`` file as reverse
+ build-dependency roots. The argument is only a regex matching parsed
+ dependency package names, not a regex matched against the full raw ``Depends``
+ field. Users should usually pass anchored package name regexes such as
+ ``^(libfoo2|libfoo1)$``.
+
+ If a matching binary maps to a source package that is not present in the
+ selected ``Sources`` indexes, ratt logs a warning and cannot schedule that
+ source for rebuild. When comparing with Ben transition pages, ensure the local
+ archive metadata includes the same relevant components, such as ``contrib``
+ for transition consumers that live there.
+
 **-json**
  Output results in JSON format (currently only works in combination with
  `-dry_run`). JSON is written to stdout; human-readable logs go to stderr. Each
@@ -143,6 +163,23 @@ Limit to direct reverse build-dependencies only::
 
   $ ratt -direct-rdeps yourpackage_*.changes
 
+Transition-aware candidate selection::
+
+  $ ratt -transition_affected '^(libfoo2|libfoo1)$' yourpackage_*.changes
+
+Choosing the transition regex:
+
+If a Ben transition tracker exists, start from its ``Affected`` expression. For
+example, convert ``Affected: .depends ~ /\b(libfoo2|libfoo1)\b/`` to
+``-transition_affected '^(libfoo2|libfoo1)$'``.
+
+If there is no tracker yet, build the regex from the old and new runtime
+library package names for the transition. Include both, since some packages may
+still depend on the old package while others already depend on the new one. Do
+not copy every binary from the ``.changes`` file into the regex. Leave out
+unrelated packages such as ``-dev``, ``-doc``, dbgsym, or utilities, unless
+they are actually part of the transition.
+
 Print dry-run result in JSON format::
 
   $ ratt -dry_run -json yourpackage_*.changes

ratt.go

@@ -104,6 +104,10 @@ var (
 		0,
 		"Set the maximum depth for reverse dependency resolution. For more details, see the --depth option in the dose-ceve(1) manpage")
 
+	transitionAffected = flag.String("transition_affected",
+		"",
+		"Select transition rebuild candidates by matching binary Depends package names against a regex")
+
 	jsonOutput = flag.Bool("json",
 		false,
 		"Output results in JSON format (currently only works in combination with -dry_run)")
@@ -225,24 +229,31 @@ func dependsOn(src control.SourceIndex, binaries map[string]bool) bool {
 	return false
 }
 
-func addReverseBuildDeps(sourcesPath string, binaries map[string]bool, rebuild map[string][]version.Version) error {
-	log.Printf("Loading sources index %q\n", sourcesPath)
+func aptIndexReader(indexPath string) (*bufio.Reader, func(), error) {
 	catFile := exec.Command("/usr/lib/apt/apt-helper",
 		"cat-file",
-		sourcesPath)
-	var s *bufio.Reader
+		indexPath)
 	if lines, err := catFile.Output(); err == nil {
-		s = bufio.NewReader(bytes.NewReader(lines))
-	} else {
-		// Fallback for older versions of apt-get. See
-		// <20160111171230.GA17291@debian.org> for context.
-		o, err := os.Open(sourcesPath)
-		if err != nil {
-			return err
-		}
-		defer o.Close()
-		s = bufio.NewReader(o)
+		return bufio.NewReader(bytes.NewReader(lines)), func() {}, nil
 	}
+
+	// Fallback for older versions of apt-get. See
+	// <20160111171230.GA17291@debian.org> for context.
+	o, err := os.Open(indexPath)
+	if err != nil {
+		return nil, nil, err
+	}
+	return bufio.NewReader(o), func() { o.Close() }, nil
+}
+
+func addReverseBuildDeps(sourcesPath string, binaries map[string]bool, rebuild map[string][]version.Version) error {
+	log.Printf("Loading sources index %q\n", sourcesPath)
+	s, cleanup, err := aptIndexReader(sourcesPath)
+	if err != nil {
+		return err
+	}
+	defer cleanup()
+
 	idx, err := control.ParseSourceIndex(s)
 	if err != nil && err != io.EOF {
 		return err
@@ -272,6 +283,91 @@ func fallback(sourcesPaths []string, binaries []string) (map[string][]version.Ve
 	return rebuild, nil
 }
 
+func binaryDependsMatchesAffected(bin control.BinaryIndex, affected *regexp.Regexp) bool {
+	depends := bin.GetDepends()
+	for _, possibility := range depends.GetAllPossibilities() {
+		if affected.MatchString(possibility.Name) {
+			return true
+		}
+	}
+	return false
+}
+
+func addTransitionAffectedSources(packagesPath string, affected *regexp.Regexp, selected map[string]struct{}) error {
+	log.Printf("Loading packages index %q\n", packagesPath)
+	p, cleanup, err := aptIndexReader(packagesPath)
+	if err != nil {
+		return err
+	}
+	defer cleanup()
+
+	idx, err := control.ParseBinaryIndex(p)
+	if err != nil && err != io.EOF {
+		return err
+	}
+
+	for _, bin := range idx {
+		if !binaryDependsMatchesAffected(bin, affected) {
+			continue
+		}
+		selected[bin.SourcePackage()] = struct{}{}
+	}
+
+	return nil
+}
+
+func addSelectedSourceVersions(sourcesPath string, selected map[string]struct{}, rebuild map[string][]version.Version) error {
+	log.Printf("Loading sources index %q\n", sourcesPath)
+	s, cleanup, err := aptIndexReader(sourcesPath)
+	if err != nil {
+		return err
+	}
+	defer cleanup()
+
+	idx, err := control.ParseSourceIndex(s)
+	if err != nil && err != io.EOF {
+		return err
+	}
+
+	for _, src := range idx {
+		if _, ok := selected[src.Package]; ok {
+			rebuild[src.Package] = append(rebuild[src.Package], src.Version)
+		}
+	}
+
+	return nil
+}
+
+func transitionAffectedSources(packagesPaths, sourcesPaths []string, affectedRegex string) (map[string][]version.Version, error) {
+	affected, err := regexp.Compile(affectedRegex)
+	if err != nil {
+		return nil, err
+	}
+
+	selected := make(map[string]struct{})
+	for _, packagesPath := range packagesPaths {
+		if err := addTransitionAffectedSources(packagesPath, affected, selected); err != nil {
+			return nil, err
+		}
+	}
+	log.Printf("Found %d source packages with binary Depends matching -transition_affected\n", len(selected))
+
+	rebuild := make(map[string][]version.Version)
+	for _, sourcesPath := range sourcesPaths {
+		if err := addSelectedSourceVersions(sourcesPath, selected, rebuild); err != nil {
+			return nil, err
+		}
+	}
+
+	for src := range selected {
+		if _, ok := rebuild[src]; !ok {
+			log.Printf("Warning: source package %q selected by -transition_affected was not found in any Sources index", src)
+		}
+	}
+
+	return rebuild, nil
+}
+
 func resolveAptListFile(indexFilePath string) (string, error) {
 	cmd := exec.Command("/usr/lib/apt/apt-helper", "cat-file", indexFilePath)
 	resolvedContent, err := cmd.Output()
@@ -636,7 +732,20 @@ func main() {
 		}
 	}
 
-	rebuild, err := reverseBuildDeps(packagesPaths, sourcesPaths, binaries)
+	var rebuild map[string][]version.Version
+	var err error
+	if *transitionAffected != "" {
+		if *directRdeps {
+			log.Printf("Warning: --direct-rdeps is ignored in -transition_affected mode")
+		}
+		if *rdepsDepth != 0 {
+			log.Printf("Warning: --rdeps-depth=%d is ignored in -transition_affected mode", *rdepsDepth)
+		}
+		log.Printf("Selecting rebuild candidates using -transition_affected=%q\n", *transitionAffected)
+		rebuild, err = transitionAffectedSources(packagesPaths, sourcesPaths, *transitionAffected)
+	} else {
+		rebuild, err = reverseBuildDeps(packagesPaths, sourcesPaths, binaries)
+	}
 	if err != nil {
 		log.Fatal(err)
 	}