// Copyright 2019 The Bazel Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package com.google.devtools.build.lib.analysis.starlark; import com.google.common.base.Joiner; import com.google.common.collect.ImmutableSet; import com.google.devtools.build.lib.actions.Artifact; import com.google.devtools.build.lib.actions.CommandLine; import com.google.devtools.build.lib.actions.CommandLineExpansionException; import com.google.devtools.build.lib.actions.CommandLines.CommandLineAndParamFileInfo; import com.google.devtools.build.lib.actions.ParamFileInfo; import com.google.devtools.build.lib.actions.ParameterFile.ParameterFileType; import com.google.devtools.build.lib.actions.SingleStringArgFormatter; import com.google.devtools.build.lib.analysis.starlark.StarlarkCustomCommandLine.ScalarArg; import com.google.devtools.build.lib.collect.nestedset.Depset; import com.google.devtools.build.lib.collect.nestedset.NestedSet; import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; import com.google.devtools.build.lib.starlarkbuildapi.CommandLineArgsApi; import com.google.errorprone.annotations.CanIgnoreReturnValue; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; import javax.annotation.Nullable; import net.starlark.java.eval.EvalException; import net.starlark.java.eval.Mutability; import net.starlark.java.eval.Printer; import net.starlark.java.eval.Sequence; import net.starlark.java.eval.Starlark; import net.starlark.java.eval.StarlarkCallable; import net.starlark.java.eval.StarlarkFunction; import net.starlark.java.eval.StarlarkSemantics; import net.starlark.java.eval.StarlarkThread; import net.starlark.java.eval.StarlarkValue; import net.starlark.java.syntax.Location; /** * Implementation of the {@code Args} Starlark type, which, in a builder-like pattern, encapsulates * the data needed to build all or part of a command line. */ public abstract class Args implements CommandLineArgsApi { private Args() { // Ensure Args subclasses are defined only in this file. } @Override public void checkHashable() throws EvalException { // Even a frozen Args is not hashable. throw Starlark.errorf("unhashable type: '%s'", Starlark.type(this)); } @Override public void repr(Printer printer) { printer.append("context.args() object"); } @Override public void debugPrint(Printer printer, StarlarkSemantics semantics) { try { printer.append(Joiner.on(" ").join(build().arguments())); } catch (CommandLineExpansionException e) { printer.append("Cannot expand command line: " + e.getMessage()); } catch (InterruptedException e) { Thread.currentThread().interrupt(); printer.append("Interrupted while expanding command line: " + e.getMessage()); } } /** * Returns the file format to use if this object's encapsulated arguments were to be written to a * param file. This value is meaningful even if {@link #getParamFileInfo} is null, as one can * force these args to be written to a param file using {@code actions.write}, even if the args * would not be written to a params file if used in normal action registration. */ public abstract ParameterFileType getParameterFileType(); /** * Returns a {@link ParamFileInfo} describing how a params file should be constructed to contain * this object's encapsulated arguments when an action is registered using this object. If a * parameter file should not be used (even under operating system arg limits), returns null. */ @Nullable public abstract ParamFileInfo getParamFileInfo(); /** * Returns a set of directory artifacts which will need to be expanded for evaluating the * encapsulated arguments during execution. */ public abstract ImmutableSet getDirectoryArtifacts(); /** Returns the command line built by this {@link Args} object. */ public abstract CommandLine build(); /** * Returns a frozen {@link Args} representation corresponding to an already-registered action. * * @param commandLineAndParamFileInfo the command line / ParamFileInfo pair that this Args should * represent * @param directoryInputs a set containing all directory artifacts of the action; {@link * Artifact#isDirectory()} must be true for each artifact in the set */ public static Args forRegisteredAction( CommandLineAndParamFileInfo commandLineAndParamFileInfo, ImmutableSet directoryInputs) { return new FrozenArgs( commandLineAndParamFileInfo.commandLine, commandLineAndParamFileInfo.paramFileInfo, directoryInputs); } /** Creates and returns a new (empty) {@link Args} object. */ public static Args newArgs(@Nullable Mutability mutability, StarlarkSemantics starlarkSemantics) { return new MutableArgs(mutability, starlarkSemantics); } /** * A frozen (immutable) representation of {@link Args}, constructed from an already-built command * line. */ @Immutable private static class FrozenArgs extends Args { private final CommandLine commandLine; private final ParamFileInfo paramFileInfo; private final ImmutableSet directoryInputs; private FrozenArgs( CommandLine commandLine, ParamFileInfo paramFileInfo, ImmutableSet directoryInputs) { this.commandLine = commandLine; this.paramFileInfo = paramFileInfo; this.directoryInputs = directoryInputs; } @Override public boolean isImmutable() { return true; // immutable but not directly hashable (though may be hashed as an element of, // say, a struct). } @Override public ImmutableSet getDirectoryArtifacts() { return directoryInputs; } @Override public CommandLine build() { return commandLine; } @Override public ParameterFileType getParameterFileType() { if (paramFileInfo != null) { return paramFileInfo.getFileType(); } else { return ParameterFileType.SHELL_QUOTED; } } @Override @Nullable public ParamFileInfo getParamFileInfo() { return paramFileInfo; } @Override public CommandLineArgsApi addArgument( Object argNameOrValue, Object value, Object format, StarlarkThread thread) throws EvalException { throw Starlark.errorf("cannot modify frozen value"); } @Override public CommandLineArgsApi addAll( Object argNameOrValue, Object values, Object mapEach, Object formatEach, Object beforeEach, Boolean omitIfEmpty, Boolean uniquify, Boolean expandDirectories, Object terminateWith, Boolean allowClosure, StarlarkThread thread) throws EvalException { throw Starlark.errorf("cannot modify frozen value"); } @Override public CommandLineArgsApi addJoined( Object argNameOrValue, Object values, String joinWith, Object mapEach, Object formatEach, Object formatJoined, Boolean omitIfEmpty, Boolean uniquify, Boolean expandDirectories, Boolean allowClosure, StarlarkThread thread) throws EvalException { throw Starlark.errorf("cannot modify frozen value"); } @Override public CommandLineArgsApi useParamsFile(String paramFileArg, Boolean useAlways) throws EvalException { // TODO(cparsons): Even "frozen" Args may need to use params files. // If we go down this path, we will need to rename this class and update the documentation // (as this class no longe behaves exactly like a frozen Args object) throw Starlark.errorf("cannot modify frozen value"); } @Override public CommandLineArgsApi setParamFileFormat(String format) throws EvalException { // TODO(cparsons): Even "frozen" Args may need to use params files. // If we go down this path, we will need to rename this class and update the documentation // (as this class no longe behaves exactly like a frozen Args object) throw Starlark.errorf("cannot modify frozen value"); } } /** Args module. */ private static class MutableArgs extends Args implements StarlarkValue, Mutability.Freezable { private final Mutability mutability; private final StarlarkCustomCommandLine.Builder commandLine; private final List> potentialDirectoryArtifacts = new ArrayList<>(); private final Set directoryArtifacts = new HashSet<>(); /** * If true, flag names and values will be grouped with '=', e.g. * * 
     *  --a=b
     *  --noc
     *  --d=e
     *
* * Further, if this is true, the ParamFileInfo will be marked 'flagsOnly', so that positional * parameters stay on the command line and the param file contains only flags. */ private boolean flagPerLine = false; // May be set explicitly once -- if unset defaults to ParameterFileType.SHELL_QUOTED. private ParameterFileType parameterFileType = null; private String flagFormatString; private boolean alwaysUseParamFile; @Override public ParameterFileType getParameterFileType() { return parameterFileType == null ? ParameterFileType.SHELL_QUOTED : parameterFileType; } @Override @Nullable public ParamFileInfo getParamFileInfo() { if (flagFormatString == null) { return null; } else { return ParamFileInfo.builder(getParameterFileType()) .setFlagFormatString(flagFormatString) .setUseAlways(alwaysUseParamFile) .setCharset(StandardCharsets.UTF_8) .setFlagsOnly(flagPerLine) .build(); } } @CanIgnoreReturnValue @Override public CommandLineArgsApi addArgument( Object argNameOrValue, Object value, Object format, StarlarkThread thread) throws EvalException { Starlark.checkMutable(this); final String argName; if (value == Starlark.UNBOUND) { value = argNameOrValue; argName = null; } else { validateArgName(argNameOrValue); argName = (String) argNameOrValue; } commandLine.recordArgStart(); if (argName != null) { commandLine.add(argName); } if (value instanceof Depset || value instanceof Sequence) { throw Starlark.errorf( "Args.add() doesn't accept vectorized arguments. Please use Args.add_all() or" + " Args.add_joined() instead."); } addScalarArg(value, format != Starlark.NONE ? (String) format : null); return this; } @CanIgnoreReturnValue @Override public CommandLineArgsApi addAll( Object argNameOrValue, Object values, Object mapEach, Object formatEach, Object beforeEach, Boolean omitIfEmpty, Boolean uniquify, Boolean expandDirectories, Object terminateWith, Boolean allowClosure, StarlarkThread thread) throws EvalException { Starlark.checkMutable(this); final String argName; if (values == Starlark.UNBOUND) { values = argNameOrValue; validateValues(values); argName = null; } else { validateArgName(argNameOrValue); argName = (String) argNameOrValue; } addVectorArg( values, argName, validateMapEach(mapEach, allowClosure), formatEach != Starlark.NONE ? (String) formatEach : null, beforeEach != Starlark.NONE ? (String) beforeEach : null, /* joinWith= */ null, /* formatJoined= */ null, omitIfEmpty, uniquify, expandDirectories, terminateWith != Starlark.NONE ? (String) terminateWith : null, thread.getCallerLocation()); return this; } @Nullable private static StarlarkCallable validateMapEach(Object fn, boolean allowClosure) throws EvalException { if (fn == Starlark.NONE) { return null; } if (fn instanceof StarlarkFunction) { StarlarkFunction sfn = (StarlarkFunction) fn; // Reject non-global functions, because arbitrary closures may cause large // analysis-phase data structures to remain live into the execution phase. // We require that the function is "global" as opposed to "not a closure" // because a global function may be closure if it refers to load bindings. // This unfortunately disallows such trivially safe non-global // functions as "lambda x: x". // See https://github.com/bazelbuild/bazel/issues/12701. if (sfn.getModule().getGlobal(sfn.getName()) != sfn && !allowClosure) { throw Starlark.errorf( "to avoid unintended retention of analysis data structures, " + "the map_each function (declared at %s) must be declared " + "by a top-level def statement", sfn.getLocation()); } } return (StarlarkCallable) fn; } @CanIgnoreReturnValue @Override public CommandLineArgsApi addJoined( Object argNameOrValue, Object values, String joinWith, Object mapEach, Object formatEach, Object formatJoined, Boolean omitIfEmpty, Boolean uniquify, Boolean expandDirectories, Boolean allowClosure, StarlarkThread thread) throws EvalException { Starlark.checkMutable(this); final String argName; if (values == Starlark.UNBOUND) { values = argNameOrValue; validateValues(values); argName = null; } else { validateArgName(argNameOrValue); argName = (String) argNameOrValue; } addVectorArg( values, argName, validateMapEach(mapEach, allowClosure), formatEach != Starlark.NONE ? (String) formatEach : null, /* beforeEach= */ null, joinWith, formatJoined != Starlark.NONE ? (String) formatJoined : null, omitIfEmpty, uniquify, expandDirectories, /* terminateWith= */ null, thread.getCallerLocation()); return this; } private void addVectorArg( Object value, String argName, StarlarkCallable mapEach, String formatEach, String beforeEach, String joinWith, String formatJoined, boolean omitIfEmpty, boolean uniquify, boolean expandDirectories, String terminateWith, Location loc) throws EvalException { validateFormatString("format_each", formatEach); validateFormatString("format_joined", formatJoined); StarlarkCustomCommandLine.VectorArg.Builder vectorArg; if (value instanceof Depset) { Depset starlarkNestedSet = (Depset) value; NestedSet nestedSet = starlarkNestedSet.getSet(); if (nestedSet.isEmpty() && omitIfEmpty) { return; } if (expandDirectories) { potentialDirectoryArtifacts.add(nestedSet); } vectorArg = new StarlarkCustomCommandLine.VectorArg.Builder(nestedSet); } else { Sequence starlarkList = (Sequence) value; if (starlarkList.isEmpty() && omitIfEmpty) { return; } if (expandDirectories) { scanForDirectories(starlarkList); } vectorArg = new StarlarkCustomCommandLine.VectorArg.Builder(starlarkList); } commandLine.recordArgStart(); vectorArg .setLocation(loc) .setArgName(argName) .setExpandDirectories(expandDirectories) .setFormatEach(formatEach) .setBeforeEach(beforeEach) .setJoinWith(joinWith) .setFormatJoined(formatJoined) .omitIfEmpty(omitIfEmpty) .uniquify(uniquify) .setTerminateWith(terminateWith) .setMapEach(mapEach); commandLine.add(vectorArg); } private void validateArgName(Object argName) throws EvalException { if (!(argName instanceof String)) { throw Starlark.errorf( "expected value of type 'string' for arg name, got '%s'", Starlark.type(argName)); } } private void validateValues(Object values) throws EvalException { if (!(values instanceof Sequence || values instanceof Depset)) { throw Starlark.errorf( "expected value of type 'sequence or depset' for values, got '%s'", Starlark.type(values)); } } private void validateFormatString(String argumentName, @Nullable String formatStr) throws EvalException { if (formatStr != null && !SingleStringArgFormatter.isValid(formatStr)) { throw Starlark.errorf( "Invalid value for parameter \"%s\": Expected string with a single \"%%s\"", argumentName); } } private void addScalarArg(Object value, String format) throws EvalException { validateNoDirectory(value); validateFormatString("format", format); if (format == null) { commandLine.add(value); } else { commandLine.add(new ScalarArg.Builder(value).setFormat(format)); } } private void validateNoDirectory(Object value) throws EvalException { if (isDirectory(value)) { throw Starlark.errorf( "Cannot add directories to Args#add since they may expand to multiple values. " + "Either use Args#add_all (if you want expansion) " + "or args.add(directory.path) (if you do not)."); } } private static boolean isDirectory(Object object) { return ((object instanceof Artifact) && ((Artifact) object).isDirectory()); } @CanIgnoreReturnValue @Override public CommandLineArgsApi useParamsFile(String paramFileArg, Boolean useAlways) throws EvalException { Starlark.checkMutable(this); if (!SingleStringArgFormatter.isValid(paramFileArg)) { throw Starlark.errorf( "Invalid value for parameter \"param_file_arg\": Expected string with a single \"%%s\"," + " got \"%s\"", paramFileArg); } this.flagFormatString = paramFileArg; this.alwaysUseParamFile = useAlways; return this; } @CanIgnoreReturnValue @Override public CommandLineArgsApi setParamFileFormat(String format) throws EvalException { Starlark.checkMutable(this); if (this.parameterFileType != null) { throw Starlark.errorf("set_param_file_format() may only be called once"); } final ParameterFileType parameterFileType; final boolean flagPerLine; switch (format) { case "shell": parameterFileType = ParameterFileType.SHELL_QUOTED; flagPerLine = false; break; case "multiline": parameterFileType = ParameterFileType.UNQUOTED; flagPerLine = false; break; case "flag_per_line": parameterFileType = ParameterFileType.UNQUOTED; flagPerLine = true; break; default: throw Starlark.errorf( "Invalid value for parameter \"format\": Expected one of \"shell\", \"multiline\"," + " \"flag_per_line\""); } this.parameterFileType = parameterFileType; this.flagPerLine = flagPerLine; return this; } private MutableArgs(@Nullable Mutability mutability, StarlarkSemantics starlarkSemantics) { this.mutability = mutability != null ? mutability : Mutability.IMMUTABLE; this.commandLine = new StarlarkCustomCommandLine.Builder(starlarkSemantics); } @Override public CommandLine build() { return commandLine.build(flagPerLine); } @Override public Mutability mutability() { return mutability; } @Override public ImmutableSet getDirectoryArtifacts() { for (NestedSet collection : potentialDirectoryArtifacts) { scanForDirectories(collection.toList()); } potentialDirectoryArtifacts.clear(); return ImmutableSet.copyOf(directoryArtifacts); } private void scanForDirectories(Iterable objects) { for (Object object : objects) { if (isDirectory(object)) { directoryArtifacts.add((Artifact) object); } } } } }