The actions executors run look like: {@code tool_pkg/mytool src/source.file * bazel-out/x86-opt/pkg/gen.file -o bazel-out/x86-opt/pkg/myout}. * *
The "x86-opt" part is a path's "configuration prefix": information describing the build * configuration of the action creating the artifact. This example shows artifacts created with * {@code --cpu=x86 --compilation_mode=opt}. * *
Executors cache actions based on their a) command line, b) input and output paths, c) input
* digests. Configuration prefixes harm caching because even if an action behaves exactly the same
* for different CPU architectures, {@code Config-stripping is an experimental feature that strips the configuration prefix from
* qualifying actions before running them, thus improving caching. "Qualifying" actions are actions
* known not to depend on the names of their input and output paths. Non-qualifying actions include
* manifest generators and compilers that store debug symbol source paths.
*
* As an experimental feature, most logic is centralized here to provide easy hooks into executor
* and action code and avoid complicating large swaths of the code base.
*
* Enable this feature by setting {@code --experimental_output_paths=strip}. This activates two
* effects:
*
* So an action is responsible for declaring that it strips paths and adjusting its command line
* accordingly. The executor is responsible for remapping action inputs and outputs to match.
*
* A lot of this work is handled generically in {@link CustomCommandLine} and related classes.
* Simple actions may be able to opt into this behavior with little more than setting {@link
* com.google.devtools.build.lib.analysis.actions.SpawnAction.Builder#stripOutputPaths(boolean)}.
* Starlark actions don't yet have API support: specific mnemonics are enabled by {@link
* com.google.devtools.build.lib.analysis.actions.StarlarkAction.Builder#stripOutputPaths(String,
* NestedSet, Artifact, BuildConfigurationValue)}.
*/
public final class PathStripper {
/**
* Creates a new {@link PathMapper} for action implementation logic to use.
*
* @param stripOutputPaths should this action strip config prefixes?
* @param starlarkMnemonic this action's mnemonic if it's a Starlark action, else null
* @param outputRoot the root path where outputs are written (e.g. "bazel-out"). Actions that
* don't strip outputs can set this to null.
*/
public static PathMapper createForAction(
boolean stripOutputPaths,
@Nullable String starlarkMnemonic,
@Nullable PathFragment outputRoot) {
if (stripOutputPaths) {
Preconditions.checkNotNull(outputRoot);
Preconditions.checkState(outputRoot.isSingleSegment());
Preconditions.checkState(!outputRoot.getPathString().contains("\\"));
}
return stripOutputPaths ? commandStripper(starlarkMnemonic, outputRoot) : PathMapper.NOOP;
}
/**
* Creates a new {@link PathMapper} for executor implementation logic to use.
*
* @param spawn the action to stage. If {@link Spawn#stripOutputPaths()} is true, paths like
* "bazel-out/k8-fastbuild/bin/foo" are reduced to "bazel-out/bin/foo". Else they're
* unchanged.
* @param outputRoot the root path where outputs are written (e.g. "bazel-out")
*/
public static PathMapper createForExecutor(Spawn spawn, PathFragment outputRoot) {
Preconditions.checkState(outputRoot.isSingleSegment());
Preconditions.checkState(!outputRoot.getPathString().contains("\\"));
return spawn.stripOutputPaths() ? actionStripper(outputRoot) : PathMapper.NOOP;
}
/** Instantiates a {@link PathMapper} that strips config prefixes from output paths. */
private static PathMapper actionStripper(PathFragment outputRoot) {
return execPath -> isOutputPath(execPath, outputRoot) ? PathStripper.strip(execPath) : execPath;
}
/** Instantiates a {@link PathMapper} that strips config prefixes from output paths. */
private static PathMapper commandStripper(
@Nullable String starlarkMnemonic, PathFragment outputRoot) {
final StringStripper argStripper =
starlarkMnemonic != null ? new StringStripper(outputRoot.getPathString()) : null;
return new PathMapper() {
@Override
public String getMappedExecPathString(ActionInput artifact) {
if (artifact instanceof DerivedArtifact) {
return PathStripper.strip(artifact);
} else {
return artifact.getExecPathString();
}
}
@Override
public PathFragment map(PathFragment execPath) {
return PathStripper.isOutputPath(execPath, outputRoot)
? PathStripper.strip(execPath)
: execPath;
}
@Override
public List Action implementation logic should use this to correctly set an action's command line. The
* executor should use this to correctly stage an action for execution.
*/
public interface PathMapper {
/**
* Returns the exec path of the input with the config part replaced if necessary.
*
* If the action should be config-stripped ({@link PathStripper}), removes "k8-fastbuild"
* from paths like "bazel-out/k8-fastbuild/foo/bar".
*
* Else returns the artifact's original exec path.
*/
default String getMappedExecPathString(ActionInput artifact) {
return map(artifact.getExecPath()).getPathString();
}
/** Same as {@link #getMappedExecPathString(ActionInput)} but for a {@link PathFragment}. */
PathFragment map(PathFragment execPath);
/**
* We don't yet have a Starlark API for mapping paths in command lines. Simple Starlark calls
* like {@code args.add(arg_name, file_path} are automatically handled. But calls that involve
* custom Starlark code require deeper API support that remains a TODO.
*
* This method hard-codes support for specific command line entries for specific Starlark
* actions that we know we want to apply stripping to.
*/
default List Rules that support path stripping can use this to help their implementation logic.
*/
public static class StringStripper {
private final Pattern pattern;
private final String outputRoot;
public StringStripper(String outputRoot) {
this.outputRoot = outputRoot;
this.pattern = stripPathsPattern(outputRoot);
}
public String strip(String str) {
return pattern.matcher(str).replaceAll(outputRoot + "/");
}
}
/**
* Returns the regex to strip output paths from a string.
*
* Supports strings with multiple output paths in arbitrary places. For example
* "/path/to/compiler bazel-out/x86-fastbuild/foo src/my.src -Dbazel-out/arm-opt/bar".
*
* Doesn't strip paths that would be non-existent without config prefixes. For example, these
* are unchanged: "bazel-out/x86-fastbuild", "bazel-out;foo", "/path/to/compiler bazel-out".
*
* @param outputRoot root segment of output paths (e.g. "bazel-out")
*/
private static Pattern stripPathsPattern(String outputRoot) {
// Match "bazel-out" followed by a slash followed by any combination of word characters, "_",
// and "-", followed by another slash. This would miss substrings like "bazel-out/k8-fastbuild".
// But those don't represent actual outputs (all outputs would have to have names beneath that
// path). So we're not trying to replace those.
return Pattern.compile(outputRoot + "/[\\w_-]+/");
}
/**
* Is this a strippable path?
*
* @param artifact artifact whose path to check
* @param outputRoot - the output tree's execPath-relative root (e.g. "bazel-out")
*/
private static boolean isOutputPath(ActionInput artifact, PathFragment outputRoot) {
// We can't simply check for DerivedArtifact. Output paths can also appear, for example, in
// ParamFileActionInput and ActionInputHelper.BasicActionInput.
return isOutputPath(artifact.getExecPath(), outputRoot);
}
/** Private utility method: Is this a strippable path? */
private static boolean isOutputPath(PathFragment pathFragment, PathFragment outputRoot) {
return pathFragment.startsWith(outputRoot);
}
/**
* Is this action safe to strip?
*
* This is distinct from whether we should strip it. An action is stripped if a) the
* action logic declares it's strippable via {@link Spawn#stripOutputPaths()} and b) it's safe to
* do that (for example, the action doesn't have two inputs in different configurations that would
* resolve to the same path if prefixes were removed).
*
* This method checks b). Action logic is responsible for considering this to set a) correctly.
*/
public static boolean isPathStrippable(
NestedSet actionInputs, PathFragment outputRoot) {
// For qualifying action types, check that no inputs or outputs would clash if paths were
// removed, e.g. "bazel-out/k8-fastbuild/foo" and "bazel-out/host/foo".
//
// A more clever algorithm could remap these with custom prefixes - "bazel-out/1/foo" and
// "bazel-out/2/foo" - if experience shows that would help.
//
// Another approach could keep host paths intact (since the "host" path prefix doesn't vary
// with configurations). While this would help more action instances qualify, it also blocks
// caching the same action in host and target configurations. This could be mitigated by
// stripping the host prefix *only* when the entire action is in the host configuration.
HashSet
*
*
*