A PackageFactory is a heavy-weight object; create them sparingly. Typically only one is needed * per client application. */ public final class PackageFactory { private static final GoogleLogger logger = GoogleLogger.forEnclosingClass(); private final RuleClassProvider ruleClassProvider; private SyscallCache syscallCache; private ForkJoinPool executor; private int maxDirectoriesToEagerlyVisitInGlobbing; private final PackageSettings packageSettings; private final PackageValidator packageValidator; private final PackageOverheadEstimator packageOverheadEstimator; private final PackageLoadingListener packageLoadingListener; /** Builder for {@link PackageFactory} instances. Intended to only be used by unit tests. */ @VisibleForTesting public abstract static class BuilderForTesting { protected PackageValidator packageValidator = PackageValidator.NOOP_VALIDATOR; protected PackageOverheadEstimator packageOverheadEstimator = PackageOverheadEstimator.NOOP_ESTIMATOR; protected boolean doChecksForTesting = true; @CanIgnoreReturnValue public BuilderForTesting disableChecks() { this.doChecksForTesting = false; return this; } @CanIgnoreReturnValue public BuilderForTesting setPackageValidator(PackageValidator packageValidator) { this.packageValidator = packageValidator; return this; } @CanIgnoreReturnValue public BuilderForTesting setPackageOverheadEstimator( PackageOverheadEstimator packageOverheadEstimator) { this.packageOverheadEstimator = packageOverheadEstimator; return this; } public abstract PackageFactory build(RuleClassProvider ruleClassProvider, FileSystem fs); } @VisibleForTesting public PackageSettings getPackageSettingsForTesting() { return packageSettings; } /** * Constructs a {@code PackageFactory} instance with a specific glob path translator and rule * factory. * *
Only intended to be called by BlazeRuntime or {@link BuilderForTesting#build}. * *
Do not call this constructor directly in tests; please use * TestConstants#PACKAGE_FACTORY_BUILDER_FACTORY_FOR_TESTING instead. */ // TODO(bazel-team): Maybe store `version` in the RuleClassProvider rather than passing it in // here? It's an extra constructor parameter that all the tests have to give, and it's only needed // so WorkspaceFactory can add an extra top-level builtin. public PackageFactory( RuleClassProvider ruleClassProvider, ForkJoinPool executorForGlobbing, PackageSettings packageSettings, PackageValidator packageValidator, PackageOverheadEstimator packageOverheadEstimator, PackageLoadingListener packageLoadingListener) { this.ruleClassProvider = ruleClassProvider; this.executor = executorForGlobbing; this.packageSettings = packageSettings; this.packageValidator = packageValidator; this.packageOverheadEstimator = packageOverheadEstimator; this.packageLoadingListener = packageLoadingListener; } /** Sets the syscalls cache used in filesystem access. */ public void setSyscallCache(SyscallCache syscallCache) { this.syscallCache = Preconditions.checkNotNull(syscallCache); } /** * Sets the max number of threads to use for globbing. * *
Internally there is a {@link ForkJoinPool} used for globbing. If the specified {@code
* globbingThreads} does not match the previous value (initial value is 100), then we {@link
* ForkJoinPool#shutdown()} the old {@link ForkJoinPool} instance and make a new one.
*/
public void setGlobbingThreads(int globbingThreads) {
if (executor == null) {
executor = makeForkJoinPool(globbingThreads);
return;
}
if (executor.getParallelism() == globbingThreads) {
return;
}
// We don't use ForkJoinPool#shutdownNow since it has a performance bug. See
// http://b/33482341#comment13.
executor.shutdown();
executor = makeForkJoinPool(globbingThreads);
}
public static ForkJoinPool makeDefaultSizedForkJoinPoolForGlobbing() {
return makeForkJoinPool(/*globbingThreads=*/ 100);
}
private static ForkJoinPool makeForkJoinPool(int globbingThreads) {
return NamedForkJoinPool.newNamedPool("globbing pool", globbingThreads);
}
/**
* Sets the number of directories to eagerly traverse on the first glob for a given package, in
* order to warm the filesystem. -1 means do no eager traversal. See {@link
* com.google.devtools.build.lib.pkgcache.PackageOptions#maxDirectoriesToEagerlyVisitInGlobbing}.
* -2 means do the eager traversal using the regular globbing infrastructure, i.e. sharing the
* globbing threads and caching the actual glob results.
*/
public void setMaxDirectoriesToEagerlyVisitInGlobbing(
int maxDirectoriesToEagerlyVisitInGlobbing) {
this.maxDirectoriesToEagerlyVisitInGlobbing = maxDirectoriesToEagerlyVisitInGlobbing;
}
/** Returns the {@link RuleClassProvider} of this {@link PackageFactory}. */
public RuleClassProvider getRuleClassProvider() {
return ruleClassProvider;
}
/** Get the PackageContext by looking up in the environment. */
public static PackageContext getContext(StarlarkThread thread) throws EvalException {
PackageContext value = thread.getThreadLocal(PackageContext.class);
if (value == null) {
// if PackageContext is missing, we're not called from a BUILD file. This happens if someone
// uses native.some_func() in the wrong place.
throw Starlark.errorf(
"The native module can be accessed only from a BUILD thread. "
+ "Wrap the function in a macro and call it from a BUILD file");
}
return value;
}
public Package.Builder newExternalPackageBuilder(
RootedPath workspacePath,
String workspaceName,
RepositoryMapping mainRepoMapping,
StarlarkSemantics starlarkSemantics) {
return Package.newExternalPackageBuilder(
packageSettings, workspacePath, workspaceName, mainRepoMapping, starlarkSemantics);
}
// This function is public only for the benefit of skyframe.PackageFunction,
// which is morally part of lib.packages, so that it can create empty packages
// in case of error before BUILD execution. Do not call it from anywhere else.
// TODO(adonovan): refactor Rule{Class,Factory}Test not to need this.
public Package.Builder newPackageBuilder(
PackageIdentifier packageId,
String workspaceName,
Optional PLEASE NOTE: the PackageContext is referred to by the StarlarkThread, but should become
* unreachable once the StarlarkThread is discarded at the end of evaluation. Please be aware of
* your memory footprint when making changes here!
*/
// TODO(adonovan): is there any reason not to merge this with Package.Builder?
public static class PackageContext {
final Package.Builder pkgBuilder;
final Globber globber;
final ExtendedEventHandler eventHandler;
@VisibleForTesting
public PackageContext(
Package.Builder pkgBuilder, Globber globber, ExtendedEventHandler eventHandler) {
this.pkgBuilder = pkgBuilder;
this.eventHandler = eventHandler;
this.globber = globber;
}
/** Returns the Label of this Package's BUILD file. */
public Label getLabel() {
return pkgBuilder.getBuildFileLabel();
}
/** Sets a Make variable. */
public void setMakeVariable(String name, String value) {
pkgBuilder.setMakeVariable(name, value);
}
/** Returns the builder of this Package. */
public Package.Builder getBuilder() {
return pkgBuilder;
}
/**
* Returns the event handler that should be used to report events happening while building this
* package.
*/
public ExtendedEventHandler getEventHandler() {
return eventHandler;
}
}
/**
* Runs final validation and administrative tasks on newly loaded package. Called by a caller of
* {@link #executeBuildFile} after this caller has fully loaded the package.
*
* @throws InvalidPackageException if the package is determined to be invalid
*/
public void afterDoneLoadingPackage(
Package pkg,
StarlarkSemantics starlarkSemantics,
long loadTimeNanos,
ExtendedEventHandler eventHandler)
throws InvalidPackageException {
OptionalLong packageOverhead = packageOverheadEstimator.estimatePackageOverhead(pkg);
packageValidator.validate(pkg, packageOverhead, eventHandler);
// Enforce limit on number of compute steps in BUILD file (b/151622307).
long maxSteps = starlarkSemantics.get(BuildLanguageOptions.MAX_COMPUTATION_STEPS);
long steps = pkg.getComputationSteps();
if (maxSteps > 0 && steps > maxSteps) {
String message =
String.format(
"BUILD file computation took %d steps, but --max_computation_steps=%d",
steps, maxSteps);
throw new InvalidPackageException(
pkg.getPackageIdentifier(),
message,
DetailedExitCode.of(
FailureDetail.newBuilder()
.setMessage(message)
.setPackageLoading(
PackageLoading.newBuilder()
.setCode(PackageLoading.Code.MAX_COMPUTATION_STEPS_EXCEEDED)
.build())
.build()));
}
packageLoadingListener.onLoadingCompleteAndSuccessful(
pkg, starlarkSemantics, loadTimeNanos, packageOverhead);
}
/**
* Populates the Package.Builder by executing the specified BUILD file.
*
* The package exists---we have parsed its BUILD file---but it may contain errors, either
* arising from Starlark evaluation (such as an array index error, or a call to a built-in
* function that fails), or reported as a side effect of a built-in function, such as rule
* instantiation, that returns normally. A partial package is nonetheless returned in both cases,
* although it may have fewer rules than expected.
*
* TODO(adonovan): do not return a partial package in case of BUILD evaluation errors. Errors
* during .bzl execution are already fatal.
*
* Do not call it from elsewhere! It is not in any meaningful sense a public API. TODO(adonovan): move PackageFunction into this package and develop a rational API.
*/
// This function is the sole entry point for package creation in production and tests. Do not add
// others! It changes often, and is exposed only for the benefit of skyframe.PackageFunction,
// which is logically part of the loading phase and should in due course be moved to lib.packages,
// but that cannot happen until Skyframe's core interfaces have been separated.
public void executeBuildFile(
Package.Builder pkgBuilder,
Program buildFileProgram,
ImmutableList It throws a {@link SyntaxError.Exception} if it discovers disallowed elements (see {@link
* DotBazelFileSyntaxChecker}).
*
* It extracts literal {@code glob(include="pattern")} patterns and adds them to {@code globs},
* or to {@code globsWithDirs} if the call had a {@code exclude_directories=0} argument.
*
* It records in {@code generatorNameByLocation} all calls of the form {@code f(name="foo",
* ...)} so that any rules instantiated during the call to {@code f} can be ascribed a "generator
* name" of {@code "foo"}.
*/
// TODO(adonovan): restructure so that this is called from the sole place that executes BUILD
// files. Also, make private; there's no reason for tests to call this directly.
public static void checkBuildSyntax(
StarlarkFile file,
Collection
* In tests, use BuildViewTestCase or PackageLoadingTestCase instead.
*
*