// Copyright 2014 The Bazel Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package com.google.devtools.build.lib.runtime; import static com.google.common.base.Preconditions.checkState; import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.devtools.build.lib.util.DetailedExitCode.DetailedExitCodeComparator.chooseMoreImportantWithFirstIfTie; import static java.nio.charset.StandardCharsets.UTF_8; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.MoreObjects; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.eventbus.EventBus; import com.google.common.eventbus.SubscriberExceptionHandler; import com.google.common.flogger.GoogleLogger; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.Uninterruptibles; import com.google.devtools.build.lib.actions.ActionKeyContext; import com.google.devtools.build.lib.analysis.BlazeDirectories; import com.google.devtools.build.lib.analysis.BlazeVersionInfo; import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider; import com.google.devtools.build.lib.analysis.ServerDirectories; import com.google.devtools.build.lib.analysis.config.BuildOptions; import com.google.devtools.build.lib.analysis.test.CoverageReportActionFactory; import com.google.devtools.build.lib.bazel.repository.downloader.Downloader; import com.google.devtools.build.lib.bugreport.BugReport; import com.google.devtools.build.lib.bugreport.BugReporter; import com.google.devtools.build.lib.bugreport.Crash; import com.google.devtools.build.lib.bugreport.CrashContext; import com.google.devtools.build.lib.buildeventstream.BuildEvent.LocalFile.LocalFileType; import com.google.devtools.build.lib.buildeventstream.BuildEventArtifactUploader; import com.google.devtools.build.lib.buildeventstream.BuildEventArtifactUploader.UploadContext; import com.google.devtools.build.lib.buildeventstream.BuildEventProtocolOptions; import com.google.devtools.build.lib.buildtool.buildevent.ProfilerStartedEvent; import com.google.devtools.build.lib.clock.BlazeClock; import com.google.devtools.build.lib.clock.Clock; import com.google.devtools.build.lib.events.Event; import com.google.devtools.build.lib.events.ExtendedEventHandler; import com.google.devtools.build.lib.events.OutputFilter; import com.google.devtools.build.lib.exec.BinTools; import com.google.devtools.build.lib.jni.JniLoader; import com.google.devtools.build.lib.packages.Package.Builder.PackageSettings; import com.google.devtools.build.lib.packages.PackageFactory; import com.google.devtools.build.lib.packages.PackageLoadingListener; import com.google.devtools.build.lib.packages.PackageOverheadEstimator; import com.google.devtools.build.lib.packages.PackageValidator; import com.google.devtools.build.lib.profiler.AutoProfiler; import com.google.devtools.build.lib.profiler.MemoryProfiler; import com.google.devtools.build.lib.profiler.ProfilePhase; import com.google.devtools.build.lib.profiler.Profiler; import com.google.devtools.build.lib.profiler.Profiler.Format; import com.google.devtools.build.lib.profiler.ProfilerTask; import com.google.devtools.build.lib.profiler.SilentCloseable; import com.google.devtools.build.lib.query2.QueryEnvironmentFactory; import com.google.devtools.build.lib.query2.engine.QueryEnvironment.QueryFunction; import com.google.devtools.build.lib.query2.query.output.OutputFormatter; import com.google.devtools.build.lib.query2.query.output.OutputFormatters; import com.google.devtools.build.lib.runtime.CommandDispatcher.LockingMode; import com.google.devtools.build.lib.runtime.proto.InvocationPolicyOuterClass.InvocationPolicy; import com.google.devtools.build.lib.server.CommandProtos.EnvironmentVariable; import com.google.devtools.build.lib.server.CommandProtos.ExecRequest; import com.google.devtools.build.lib.server.FailureDetails.FailureDetail; import com.google.devtools.build.lib.server.FailureDetails.Filesystem; import com.google.devtools.build.lib.server.GrpcServerImpl; import com.google.devtools.build.lib.server.PidFileWatcher; import com.google.devtools.build.lib.server.RPCServer; import com.google.devtools.build.lib.server.ShutdownHooks; import com.google.devtools.build.lib.server.signal.InterruptSignalHandler; import com.google.devtools.build.lib.util.AbruptExitException; import com.google.devtools.build.lib.util.CustomExitCodePublisher; import com.google.devtools.build.lib.util.CustomFailureDetailPublisher; import com.google.devtools.build.lib.util.DebugLoggerConfigurator; import com.google.devtools.build.lib.util.DetailedExitCode; import com.google.devtools.build.lib.util.ExitCode; import com.google.devtools.build.lib.util.InterruptedFailureDetails; import com.google.devtools.build.lib.util.LoggingUtil; import com.google.devtools.build.lib.util.Pair; import com.google.devtools.build.lib.util.ProcessUtils; import com.google.devtools.build.lib.util.TestType; import com.google.devtools.build.lib.util.ThreadUtils; import com.google.devtools.build.lib.util.io.OutErr; import com.google.devtools.build.lib.vfs.FileSystem; import com.google.devtools.build.lib.vfs.FileSystemUtils; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; import com.google.devtools.build.lib.worker.WorkerMetricsCollector; import com.google.devtools.common.options.CommandNameCache; import com.google.devtools.common.options.InvocationPolicyParser; import com.google.devtools.common.options.OptionDefinition; import com.google.devtools.common.options.OptionPriority.PriorityCategory; import com.google.devtools.common.options.OptionsBase; import com.google.devtools.common.options.OptionsParser; import com.google.devtools.common.options.OptionsParsingException; import com.google.devtools.common.options.OptionsParsingResult; import com.google.devtools.common.options.OptionsProvider; import com.google.devtools.common.options.TriState; import com.google.errorprone.annotations.CanIgnoreReturnValue; import java.io.File; import java.io.IOException; import java.io.OutputStream; import java.io.PrintStream; import java.lang.reflect.Type; import java.nio.charset.StandardCharsets; import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; import java.util.EnumSet; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.UUID; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; import java.util.function.Supplier; import java.util.logging.Handler; import java.util.logging.LogRecord; import java.util.logging.Logger; import javax.annotation.Nullable; import net.starlark.java.eval.EvalException; /** * The BlazeRuntime class encapsulates the immutable configuration of the current instance. These * runtime settings and services are available to most parts of any Blaze application for the * duration of the batch run or server lifetime. * *

The parts specific to the current command are stored in {@link CommandEnvironment}. */ public final class BlazeRuntime implements BugReport.BlazeRuntimeInterface { private static final GoogleLogger logger = GoogleLogger.forEnclosingClass(); private final FileSystem fileSystem; private final ImmutableList blazeModules; private final Map commandMap = new LinkedHashMap<>(); private final Clock clock; private final Runnable abruptShutdownHandler; private final PackageFactory packageFactory; private final ConfiguredRuleClassProvider ruleClassProvider; // For bazel info. private final ImmutableMap infoItems; // For bazel query. private final QueryEnvironmentFactory queryEnvironmentFactory; private final ImmutableList queryFunctions; private final ImmutableList queryOutputFormatters; private final AtomicReference storedExitCode = new AtomicReference<>(); // TODO(b/1030062): If multiple commands can ever run simultaneously, this should be a set of // command environments, with environments removed in some after-command hook. // Null if a command is not in progress. @Nullable private volatile CommandEnvironment env = null; // We pass this through here to make it available to the MasterLogWriter. private final OptionsParsingResult startupOptionsProvider; private final ProjectFile.Provider projectFileProvider; private final QueryRuntimeHelper.Factory queryRuntimeHelperFactory; private final InvocationPolicy moduleInvocationPolicy; private final SubscriberExceptionHandler eventBusExceptionHandler; private final BugReporter bugReporter; private final String productName; private final BuildEventArtifactUploaderFactoryMap buildEventArtifactUploaderFactoryMap; private final ActionKeyContext actionKeyContext; private final ImmutableMap authHeadersProviderMap; @Nullable private final RepositoryRemoteExecutorFactory repositoryRemoteExecutorFactory; private final Supplier downloaderSupplier; // Workspace state (currently exactly one workspace per server) private BlazeWorkspace workspace; private BlazeRuntime( FileSystem fileSystem, QueryEnvironmentFactory queryEnvironmentFactory, ImmutableList queryFunctions, ImmutableList queryOutputFormatters, PackageFactory pkgFactory, ConfiguredRuleClassProvider ruleClassProvider, ImmutableMap infoItems, ActionKeyContext actionKeyContext, Clock clock, Runnable abruptShutdownHandler, OptionsParsingResult startupOptionsProvider, ImmutableList blazeModules, SubscriberExceptionHandler eventBusExceptionHandler, BugReporter bugReporter, ProjectFile.Provider projectFileProvider, QueryRuntimeHelper.Factory queryRuntimeHelperFactory, InvocationPolicy moduleInvocationPolicy, Iterable commands, String productName, BuildEventArtifactUploaderFactoryMap buildEventArtifactUploaderFactoryMap, ImmutableMap authHeadersProviderMap, RepositoryRemoteExecutorFactory repositoryRemoteExecutorFactory, Supplier downloaderSupplier) { // Server state this.fileSystem = fileSystem; this.blazeModules = blazeModules; overrideCommands(commands); this.packageFactory = pkgFactory; this.projectFileProvider = projectFileProvider; this.queryRuntimeHelperFactory = queryRuntimeHelperFactory; this.moduleInvocationPolicy = moduleInvocationPolicy; this.ruleClassProvider = ruleClassProvider; this.infoItems = infoItems; this.actionKeyContext = actionKeyContext; this.clock = clock; this.abruptShutdownHandler = abruptShutdownHandler; this.startupOptionsProvider = startupOptionsProvider; this.queryEnvironmentFactory = queryEnvironmentFactory; this.queryFunctions = queryFunctions; this.queryOutputFormatters = queryOutputFormatters; this.eventBusExceptionHandler = eventBusExceptionHandler; this.bugReporter = bugReporter; CommandNameCache.CommandNameCacheInstance.INSTANCE.setCommandNameCache( new CommandNameCacheImpl(commandMap)); this.productName = productName; this.buildEventArtifactUploaderFactoryMap = buildEventArtifactUploaderFactoryMap; this.authHeadersProviderMap = Preconditions.checkNotNull(authHeadersProviderMap, "authHeadersProviderMap"); this.repositoryRemoteExecutorFactory = repositoryRemoteExecutorFactory; this.downloaderSupplier = downloaderSupplier; } public BlazeWorkspace initWorkspace(BlazeDirectories directories, BinTools binTools) throws AbruptExitException { Preconditions.checkState(this.workspace == null); WorkspaceBuilder builder = new WorkspaceBuilder(directories, binTools); for (BlazeModule module : blazeModules) { module.workspaceInit(this, directories, builder); } this.workspace = builder.build(this, packageFactory, eventBusExceptionHandler); return workspace; } @Nullable public CoverageReportActionFactory getCoverageReportActionFactory( OptionsProvider commandOptions) { CoverageReportActionFactory firstFactory = null; for (BlazeModule module : blazeModules) { CoverageReportActionFactory factory = module.getCoverageReportFactory(commandOptions); if (factory != null) { Preconditions.checkState( firstFactory == null, "only one Bazel Module can have a Coverage Report Factory"); firstFactory = factory; } } return firstFactory; } /** * Adds the given command under the given name to the map of commands. * * @throws AssertionError if the name is already used by another command. */ private void addCommand(BlazeCommand command) { String name = command.getClass().getAnnotation(Command.class).name(); if (commandMap.containsKey(name)) { throw new IllegalStateException("Command name or alias " + name + " is already used."); } commandMap.put(name, command); } @VisibleForTesting public void overrideCommands(Iterable commands) { commandMap.clear(); for (BlazeCommand command : commands) { addCommand(command); } } public InvocationPolicy getModuleInvocationPolicy() { return moduleInvocationPolicy; } private BuildEventArtifactUploader newUploader( CommandEnvironment env, String buildEventUploadStrategy) throws IOException { return buildEventArtifactUploaderFactoryMap.select(buildEventUploadStrategy).create(env); } /** Configure profiling based on the provided options. */ ProfilerStartedEvent initProfiler( boolean tracerEnabled, ExtendedEventHandler eventHandler, BlazeWorkspace workspace, CommonCommandOptions options, BuildEventProtocolOptions bepOptions, CommandEnvironment env, long execStartTimeNanos, long waitTimeInMs) { OutputStream out = null; boolean recordFullProfilerData = options.recordFullProfilerData; ImmutableSet.Builder profiledTasksBuilder = ImmutableSet.builder(); Profiler.Format format = Format.JSON_TRACE_FILE_FORMAT; Path profilePath = null; String profileName = null; UploadContext streamingContext = null; try { if (tracerEnabled) { if (options.profilePath == null) { profileName = "command.profile.gz"; format = Format.JSON_TRACE_FILE_COMPRESSED_FORMAT; if (bepOptions != null && bepOptions.streamingLogFileUploads) { BuildEventArtifactUploader buildEventArtifactUploader = newUploader(env, bepOptions.buildEventUploadStrategy); streamingContext = buildEventArtifactUploader.startUpload(LocalFileType.LOG, null); out = streamingContext.getOutputStream(); } else { profilePath = workspace.getOutputBase().getRelative(profileName); out = profilePath.getOutputStream(); } } else { format = options.profilePath.toString().endsWith(".gz") ? Format.JSON_TRACE_FILE_COMPRESSED_FORMAT : Format.JSON_TRACE_FILE_FORMAT; profilePath = workspace.getWorkspace().getRelative(options.profilePath); out = profilePath.getOutputStream(/* append= */ false, /* internal= */ true); } if (profilePath != null && options.announceProfilePath) { eventHandler.handle(Event.info("Writing tracer profile to '" + profilePath + "'")); } for (ProfilerTask profilerTask : ProfilerTask.values()) { if (!profilerTask.isVfs() // CRITICAL_PATH corresponds to writing the file. && profilerTask != ProfilerTask.CRITICAL_PATH && profilerTask != ProfilerTask.SKYFUNCTION) { profiledTasksBuilder.add(profilerTask); } } profiledTasksBuilder.addAll(options.additionalProfileTasks); if (options.recordFullProfilerData) { profiledTasksBuilder.addAll(EnumSet.allOf(ProfilerTask.class)); } } else if (options.alwaysProfileSlowOperations) { recordFullProfilerData = false; out = null; for (ProfilerTask profilerTask : ProfilerTask.values()) { if (profilerTask.collectsSlowestInstances()) { profiledTasksBuilder.add(profilerTask); } } } ImmutableSet profiledTasks = profiledTasksBuilder.build(); if (!profiledTasks.isEmpty()) { if (options.slimProfile && options.includePrimaryOutput) { eventHandler.handle( Event.warn( "Enabling both --slim_profile and" + " --experimental_profile_include_primary_output: the \"out\" field" + " will be omitted in merged actions.")); } Profiler profiler = Profiler.instance(); WorkerMetricsCollector workerMetricsCollector = WorkerMetricsCollector.instance(); workerMetricsCollector.setClock(clock); profiler.start( profiledTasks, out, format, workspace.getOutputBase().toString(), env.getCommandId(), recordFullProfilerData, clock, execStartTimeNanos, options.slimProfile, options.includePrimaryOutput, options.profileIncludeTargetLabel, options.alwaysProfileSlowOperations, options.collectWorkerDataInProfiler, options.collectLoadAverageInProfiler, options.collectSystemNetworkUsage, options.collectPressureStallIndicators, options.collectResourceEstimation, env.getLocalResourceManager(), WorkerMetricsCollector.instance(), bugReporter); // Instead of logEvent() we're calling the low level function to pass the timings we took in // the launcher. We're setting the INIT phase marker so that it follows immediately the // LAUNCH phase. long startupTimeNanos = options.startupTime * 1000000L; long waitTimeNanos = waitTimeInMs * 1000000L; long clientStartTimeNanos = execStartTimeNanos - startupTimeNanos - waitTimeNanos; profiler.logSimpleTaskDuration( clientStartTimeNanos, Duration.ofNanos(startupTimeNanos), ProfilerTask.PHASE, ProfilePhase.LAUNCH.description); if (options.extractDataTime > 0) { profiler.logSimpleTaskDuration( clientStartTimeNanos, Duration.ofMillis(options.extractDataTime), ProfilerTask.PHASE, "Extracting Bazel binary"); } if (options.waitTime > 0) { profiler.logSimpleTaskDuration( clientStartTimeNanos, Duration.ofMillis(options.waitTime), ProfilerTask.PHASE, "Blocking on busy Bazel server (in client)"); } if (waitTimeInMs > 0) { profiler.logSimpleTaskDuration( clientStartTimeNanos + startupTimeNanos, Duration.ofMillis(waitTimeInMs), ProfilerTask.PHASE, "Blocking on busy Bazel server (in server)"); } profiler.logSimpleTaskDuration( execStartTimeNanos, Duration.ZERO, ProfilerTask.PHASE, ProfilePhase.INIT.description); } } catch (IOException e) { eventHandler.handle(Event.error("Error while creating profile file: " + e.getMessage())); } return new ProfilerStartedEvent(profileName, profilePath, streamingContext); } public FileSystem getFileSystem() { return fileSystem; } public BlazeWorkspace getWorkspace() { return workspace; } public ActionKeyContext getActionKeyContext() { return actionKeyContext; } /** The directory in which blaze stores the server state - that is, the socket file and a log. */ private Path getServerDirectory() { return workspace.getDirectories().getOutputBase().getChild("server"); } /** * Returns the {@link QueryEnvironmentFactory} that should be used to create a {@link * com.google.devtools.build.lib.query2.common.AbstractBlazeQueryEnvironment}, whenever one is * needed. */ public QueryEnvironmentFactory getQueryEnvironmentFactory() { return queryEnvironmentFactory; } public ImmutableList getQueryFunctions() { return queryFunctions; } public ImmutableList getQueryOutputFormatters() { return queryOutputFormatters; } /** Returns the package factory. */ public PackageFactory getPackageFactory() { return packageFactory; } /** Returns the rule class provider. */ public ConfiguredRuleClassProvider getRuleClassProvider() { return ruleClassProvider; } public ImmutableMap getInfoItems() { return infoItems; } public Iterable getBlazeModules() { return blazeModules; } /** * Returns the first module that is an instance of a given class or interface. * * @param moduleClass a class or interface that we want to match to a module * @param the type of the module's class * @return a module that is an instance of this class or interface */ @SuppressWarnings("unchecked") public T getBlazeModule(Class moduleClass) { for (BlazeModule module : blazeModules) { if (moduleClass.isInstance(module)) { return (T) module; } } return null; } /** * Returns a provider for project file objects. Can be null if no such provider was set by any of * the modules. */ @Nullable public ProjectFile.Provider getProjectFileProvider() { return projectFileProvider; } public QueryRuntimeHelper.Factory getQueryRuntimeHelperFactory() { return queryRuntimeHelperFactory; } /** * Hook method called by the BlazeCommandDispatcher prior to the dispatch of each command. * * @param options The CommonCommandOptions used by every command. */ void beforeCommand(CommandEnvironment env, CommonCommandOptions options) { this.env = env; if (options.memoryProfilePath != null) { Path memoryProfilePath = env.getWorkingDirectory().getRelative(options.memoryProfilePath); MemoryProfiler.instance() .setStableMemoryParameters(options.memoryProfileStableHeapParameters); try { MemoryProfiler.instance().start(memoryProfilePath.getOutputStream()); } catch (IOException e) { env.getReporter() .handle(Event.error("Error while creating memory profile file: " + e.getMessage())); } } } @Override public void cleanUpForCrash(DetailedExitCode exitCode) { logger.atInfo().log("Cleaning up in crash: %s", exitCode); if (declareExitCode(exitCode)) { CommandEnvironment localEnv = env; if (localEnv != null) { localEnv.notifyOnCrash( productName + " is crashing: " + exitCode.getFailureDetail().getMessage()); } // Only try to publish events if we won the exit code race. Otherwise someone else is already // exiting for us. if (workspace == null) { return; // A crash during server startup. } EventBus eventBus = workspace.getSkyframeExecutor().getEventBus(); if (eventBus != null) { workspace .getSkyframeExecutor() .postLoggingStatsWhenCrashing( new ExtendedEventHandler() { @Override public void post(Postable obj) { eventBus.post(obj); } @Override public void handle(Event event) {} }); eventBus.post(new CrashEvent()); eventBus.post(new CommandCompleteEvent(exitCode)); } } // We don't call #shutDown() here because all it does is shut down the modules, and who knows if // they can be trusted. Instead, we call runtime#shutdownOnCrash() which attempts to cleanly // shut down those modules that might have something pending to do as a best-effort operation. shutDownModulesOnCrash(exitCode); } @Nullable public DetailedExitCode getCrashExitCode() { return storedExitCode.get(); } private boolean declareExitCode(DetailedExitCode detailedExitCode) { return storedExitCode.compareAndSet(null, detailedExitCode); } /** * Posts the {@link CommandCompleteEvent}, so that listeners can tidy up. Called by {@link * #afterCommand}, and by BugReport when crashing from an exception in an async thread. * *

Returns null if {@code exitCode} was registered as the exit code, and the {@link * DetailedExitCode} to use if another thread already registered an exit code. */ @Nullable private DetailedExitCode notifyCommandComplete(DetailedExitCode exitCode) { if (!declareExitCode(exitCode)) { // This command has already been called, presumably because there is a race between the main // thread and a worker thread that crashed. Don't try to arbitrate the dispute. If the main // thread won the race (unlikely, but possible), this may be incorrectly logged as a success. return storedExitCode.get(); } workspace.getSkyframeExecutor().getEventBus().post(new CommandCompleteEvent(exitCode)); return null; } /** * Hook method called by the BlazeCommandDispatcher after the dispatch of each command. Returns a * new exit code in case exceptions were encountered during cleanup. */ @VisibleForTesting public BlazeCommandResult afterCommand(CommandEnvironment env, BlazeCommandResult commandResult) { this.env = null; // Remove any filters that the command might have added to the reporter. env.getReporter().setOutputFilter(OutputFilter.OUTPUT_EVERYTHING); DetailedExitCode moduleExitCode = null; for (BlazeModule module : blazeModules) { try (SilentCloseable c = Profiler.instance().profile(module + ".afterCommand")) { module.afterCommand(); } catch (AbruptExitException e) { env.getReporter().handle(Event.error(e.getMessage())); logger.atWarning().withCause(e).log("While running afterCommand() on %s", module); moduleExitCode = chooseMoreImportantWithFirstIfTie(moduleExitCode, e.getDetailedExitCode()); } } env.getEventBus().post(new AfterCommandEvent()); // Wipe the dependency graph if requested. Note that this method always runs at the end of // a commands unless the server crashes, in which case no inmemory state will linger for the // next build anyway. CommonCommandOptions commonOptions = env.getOptions().getOptions(CommonCommandOptions.class); if (!commonOptions.keepStateAfterBuild) { workspace.getSkyframeExecutor().resetEvaluator(); } // Build-related commands already call this hook in BuildTool#stopRequest, but non-build // commands might also need to notify the SkyframeExecutor. It's called in #stopRequest so that // timing metrics for builds can be more accurate (since this call can be slow). try { workspace.getSkyframeExecutor().notifyCommandComplete(env.getReporter()); } catch (InterruptedException e) { logger.atInfo().withCause(e).log("Interrupted in afterCommand"); moduleExitCode = chooseMoreImportantWithFirstIfTie( moduleExitCode, InterruptedFailureDetails.detailedExitCode("executor completion interrupted")); Thread.currentThread().interrupt(); } BlazeCommandResult finalCommandResult; if (!commandResult.getExitCode().isInfrastructureFailure() && moduleExitCode != null) { finalCommandResult = BlazeCommandResult.detailedExitCode(moduleExitCode); } else { finalCommandResult = commandResult; } DetailedExitCode otherThreadWonExitCode = notifyCommandComplete(finalCommandResult.getDetailedExitCode()); if (otherThreadWonExitCode != null) { finalCommandResult = BlazeCommandResult.detailedExitCode(otherThreadWonExitCode); } env.getBlazeWorkspace().clearEventBus(); for (BlazeModule module : blazeModules) { try (SilentCloseable closeable = Profiler.instance().profile(module + ".commandComplete")) { module.commandComplete(); } } env.getReporter().clearEventBus(); actionKeyContext.clear(); DebugLoggerConfigurator.flushServerLog(); storedExitCode.set(null); return BlazeCommandResult.withResponseExtensions( finalCommandResult, env.getResponseExtensions(), commonOptions.keepStateAfterBuild); } /** * Returns the Clock-instance used for the entire build. Before, individual classes (such as * Profiler) used to specify the type of clock (e.g. EpochClock) they wanted to use. This made it * difficult to get Blaze working on Windows as some of the clocks available for Linux aren't * (directly) available on Windows. Setting the Blaze-wide clock upon construction of BlazeRuntime * allows injecting whatever Clock instance should be used from BlazeMain. * * @return The Blaze-wide clock */ public Clock getClock() { return clock; } /** * Returns the {@link BugReporter} that should be used when filing bug reports, if possible. Use * this in preference to {@link BugReport#sendBugReport} for ease of testing codepaths that file * bug reports. */ public BugReporter getBugReporter() { return bugReporter; } public OptionsParsingResult getStartupOptionsProvider() { return startupOptionsProvider; } public Map getCommandMap() { return commandMap; } /** Invokes {@link BlazeModule#blazeShutdown()} on all registered modules. */ public void shutdown() { try { for (BlazeModule module : blazeModules) { module.blazeShutdown(); } } finally { DebugLoggerConfigurator.flushServerLog(); } } public void prepareForAbruptShutdown() { if (abruptShutdownHandler != null) { abruptShutdownHandler.run(); } } /** Invokes {@link BlazeModule#blazeShutdownOnCrash} on all registered modules. */ private void shutDownModulesOnCrash(DetailedExitCode exitCode) { // TODO(b/167592709): remove verbose logging when bug resolved. logger.atInfo().log("Shutting down modules on crash: %s", blazeModules); try { for (BlazeModule module : blazeModules) { logger.atInfo().log("Shutting down %s on crash", module); module.blazeShutdownOnCrash(exitCode); } } finally { DebugLoggerConfigurator.flushServerLog(); } } /** Creates a BuildOptions class for the given options taken from an {@link OptionsProvider}. */ public BuildOptions createBuildOptions(OptionsProvider optionsProvider) { return BuildOptions.of( ruleClassProvider.getFragmentRegistry().getOptionsClasses(), optionsProvider); } /** * Main method for the Blaze server startup. Note: This method logs exceptions to remote servers. * Do not add this to a unittest. */ public static void main(Iterable> moduleClasses, String[] args) { setupUncaughtHandlerAtStartup(args); List modules = createModules(moduleClasses); // blaze.cc will put --batch first if the user set it. if (args.length >= 1 && args[0].equals("--batch")) { // Run Blaze in batch mode. System.exit(batchMain(modules, args)); } logger.atInfo().log( "Starting Bazel server with %s, args %s", maybeGetPidString(), Arrays.toString(args)); try { // Run Blaze in server mode. System.exit(serverMain(modules, OutErr.SYSTEM_OUT_ERR, args)); } catch (RuntimeException | Error e) { // A definite bug... Crash crash = Crash.from(e); BugReport.handleCrash(crash, CrashContext.keepAlive().withArgs(args)); System.exit(crash.getDetailedExitCode().getExitCode().getNumericExitCode()); } } @VisibleForTesting public static List createModules( Iterable> moduleClasses) { ImmutableList.Builder result = ImmutableList.builder(); for (Class moduleClass : moduleClasses) { try { BlazeModule module = moduleClass.getConstructor().newInstance(); result.add(module); } catch (Throwable e) { throw new IllegalStateException("Cannot instantiate module " + moduleClass.getName(), e); } } return result.build(); } /** Command line options split in to two parts: startup options and everything else. */ @VisibleForTesting static class CommandLineOptions { private final ImmutableList startupArgs; private final ImmutableList otherArgs; CommandLineOptions(ImmutableList startupArgs, ImmutableList otherArgs) { this.startupArgs = Preconditions.checkNotNull(startupArgs); this.otherArgs = Preconditions.checkNotNull(otherArgs); } public ImmutableList getStartupArgs() { return startupArgs; } public ImmutableList getOtherArgs() { return otherArgs; } } /** * Splits given options into two lists - arguments matching options defined in this class and * everything else, while preserving order in each list. * *

Note that this method relies on the startup options always being in the --flag=ARG * form (instead of --flag ARG). This is enforced by GetArgumentArray() * in blaze.cc by reconstructing the startup options from their parsed * versions instead of using argv verbatim. */ static CommandLineOptions splitStartupOptions(Iterable modules, String... args) { List prefixes = new ArrayList<>(); List startupOptions = Lists.newArrayList(); for (Class defaultOptions : BlazeCommandUtils.getStartupOptions(modules)) { startupOptions.addAll(OptionsParser.getOptionDefinitions(defaultOptions)); } for (OptionDefinition optionDefinition : startupOptions) { Type optionType = optionDefinition.getField().getType(); prefixes.add("--" + optionDefinition.getOptionName()); if (optionType == boolean.class || optionType == TriState.class) { prefixes.add("--no" + optionDefinition.getOptionName()); } } List startupArgs = new ArrayList<>(); List otherArgs = Lists.newArrayList(args); for (Iterator argi = otherArgs.iterator(); argi.hasNext(); ) { String arg = argi.next(); if (!arg.startsWith("--")) { break; // stop at command - all startup options would be specified before it. } for (String prefix : prefixes) { if (arg.startsWith(prefix)) { startupArgs.add(arg); argi.remove(); break; } } } return new CommandLineOptions( ImmutableList.copyOf(startupArgs), ImmutableList.copyOf(otherArgs)); } @Nullable private static String getSlowInterruptMessageSuffix(Iterable modules) { String slowInterruptMessageSuffix = null; for (BlazeModule module : modules) { String message = module.getSlowThreadInterruptMessageSuffix(); if (message != null) { checkState( slowInterruptMessageSuffix == null, "Two messages: %s %s (%s)", slowInterruptMessageSuffix, message, module); slowInterruptMessageSuffix = message; } } return slowInterruptMessageSuffix; } private static InterruptSignalHandler captureSigint(@Nullable String slowInterruptMessage) { Thread mainThread = Thread.currentThread(); AtomicInteger numInterrupts = new AtomicInteger(); Runnable interruptWatcher = () -> { int count = 0; // Not an actual infinite loop because it's run in a daemon thread. while (true) { count++; Uninterruptibles.sleepUninterruptibly(10, TimeUnit.SECONDS); logger.atWarning().log("Slow interrupt number %d in batch mode", count); ThreadUtils.warnAboutSlowInterrupt(slowInterruptMessage); } }; return new InterruptSignalHandler() { @Override public void run() { logger.atInfo().log("User interrupt"); OutErr.SYSTEM_OUT_ERR.printErrLn("Bazel received an interrupt"); mainThread.interrupt(); int curNumInterrupts = numInterrupts.incrementAndGet(); if (curNumInterrupts == 1) { Thread interruptWatcherThread = new Thread(interruptWatcher, "interrupt-watcher"); interruptWatcherThread.setDaemon(true); interruptWatcherThread.start(); } else if (curNumInterrupts == 2) { logger.atWarning().log("Second --batch interrupt: Reverting to JVM SIGINT handler"); uninstall(); } } }; } /** * A main method that runs blaze commands in batch mode. The return value indicates the desired * exit status of the program. */ private static int batchMain(Iterable modules, String[] args) { InterruptSignalHandler signalHandler = captureSigint(getSlowInterruptMessageSuffix(modules)); CommandLineOptions commandLineOptions = splitStartupOptions(modules, args); logger.atInfo().log( "Running Bazel in batch mode with %s, startup args %s", maybeGetPidString(), commandLineOptions.getStartupArgs()); BlazeRuntime runtime; InvocationPolicy policy; try { runtime = newRuntime(modules, commandLineOptions.getStartupArgs(), null); policy = InvocationPolicyParser.parsePolicy( runtime.startupOptionsProvider.getOptions(BlazeServerStartupOptions.class) .invocationPolicy); } catch (OptionsParsingException e) { OutErr.SYSTEM_OUT_ERR.printErrLn(e.getMessage()); return ExitCode.COMMAND_LINE_ERROR.getNumericExitCode(); } catch (AbruptExitException e) { OutErr.SYSTEM_OUT_ERR.printErrLn(e.getMessage()); return e.getExitCode().getNumericExitCode(); } ImmutableList.Builder> startupOptionsFromCommandLine = ImmutableList.builder(); for (String option : commandLineOptions.getStartupArgs()) { startupOptionsFromCommandLine.add(new Pair<>("", option)); } BlazeCommandDispatcher dispatcher = new BlazeCommandDispatcher(runtime, BlazeCommandDispatcher.UNKNOWN_SERVER_PID); boolean shutdownDone = false; try { logger.atInfo().log( "%s", SafeRequestLogging.getRequestLogString(commandLineOptions.getOtherArgs())); BlazeCommandResult result = dispatcher.exec( policy, commandLineOptions.getOtherArgs(), OutErr.SYSTEM_OUT_ERR, LockingMode.ERROR_OUT, "batch client", runtime.clock.currentTimeMillis(), Optional.of(startupOptionsFromCommandLine.build()), /*commandExtensions=*/ ImmutableList.of()); if (result.getExecRequest() == null) { // Simple case: we are given an exit code return result.getExitCode().getNumericExitCode(); } // Not so simple case: we need to execute a binary on shutdown. exec() is not accessible from // Java and is impossible on Windows in any case, so we just execute the binary after getting // out of the way as completely as possible and forward its exit code. // When this code is executed, no locks are held: the client lock is released by the client // before it executes any command and the server lock is handled by BlazeCommandDispatcher, // whose job is done by the time we get here. runtime.shutdown(); dispatcher.shutdown(); shutdownDone = true; signalHandler.uninstall(); ExecRequest request = result.getExecRequest(); String[] argv = new String[request.getArgvCount()]; for (int i = 0; i < argv.length; i++) { argv[i] = request.getArgv(i).toString(StandardCharsets.ISO_8859_1); } String workingDirectory = request.getWorkingDirectory().toString(StandardCharsets.ISO_8859_1); try { ProcessBuilder process = new ProcessBuilder().command(argv).directory(new File(workingDirectory)).inheritIO(); for (int i = 0; i < request.getEnvironmentVariableCount(); i++) { EnvironmentVariable variable = request.getEnvironmentVariable(i); process .environment() .put( variable.getName().toString(StandardCharsets.ISO_8859_1), variable.getValue().toString(StandardCharsets.ISO_8859_1)); } return process.start().waitFor(); } catch (IOException e) { // We are in batch mode, thus, stdout/stderr are the same as that of the client. System.err.println("Cannot execute process for 'run' command: " + e.getMessage()); logger.atSevere().withCause(e).log("Exception while executing binary from 'run' command"); return ExitCode.LOCAL_ENVIRONMENTAL_ERROR.getNumericExitCode(); } } catch (InterruptedException e) { // This is almost main(), so it's okay to just swallow it. We are exiting soon. return ExitCode.INTERRUPTED.getNumericExitCode(); } finally { if (!shutdownDone) { runtime.shutdown(); dispatcher.shutdown(); } } } /** * A main method that does not send email. The return value indicates the desired exit status of * the program. */ private static int serverMain(Iterable modules, OutErr outErr, String[] args) { InterruptSignalHandler sigintHandler = null; try { AtomicReference rpcServerRef = new AtomicReference<>(); Runnable prepareForAbruptShutdown = () -> rpcServerRef.get().prepareForAbruptShutdown(); BlazeRuntime runtime = newRuntime(modules, Arrays.asList(args), prepareForAbruptShutdown); // server.pid was written in the C++ launcher after fork() but before exec(). The client only // accesses the pid file after connecting to the socket which ensures that it gets the correct // pid value. Path pidFile = runtime.getServerDirectory().getRelative("server.pid.txt"); int serverPid = readPidFile(pidFile); PidFileWatcher pidFileWatcher = new PidFileWatcher(pidFile, serverPid); pidFileWatcher.start(); ShutdownHooks shutdownHooks = ShutdownHooks.createAndRegister(); shutdownHooks.deleteAtExit(pidFile); BlazeCommandDispatcher dispatcher = new BlazeCommandDispatcher(runtime, serverPid); BlazeServerStartupOptions startupOptions = runtime.startupOptionsProvider.getOptions(BlazeServerStartupOptions.class); RPCServer rpcServer = GrpcServerImpl.create( dispatcher, shutdownHooks, pidFileWatcher, runtime.clock, startupOptions.commandPort, runtime.getServerDirectory(), serverPid, startupOptions.maxIdleSeconds, startupOptions.shutdownOnLowSysMem, startupOptions.idleServerTasks, getSlowInterruptMessageSuffix(modules)); rpcServerRef.set(rpcServer); // Register the signal handler. sigintHandler = new InterruptSignalHandler() { @Override public void run() { logger.atSevere().log("User interrupt"); rpcServer.interrupt(); } }; rpcServer.serve(); runtime.shutdown(); dispatcher.shutdown(); return ExitCode.SUCCESS.getNumericExitCode(); } catch (OptionsParsingException e) { outErr.printErrLn(e.getMessage()); return ExitCode.COMMAND_LINE_ERROR.getNumericExitCode(); } catch (AbruptExitException e) { outErr.printErrLn(e.getMessage()); e.printStackTrace(new PrintStream(outErr.getErrorStream(), true)); FailureDetail failureDetail = e.getDetailedExitCode().getFailureDetail(); if (failureDetail != null) { CustomFailureDetailPublisher.maybeWriteFailureDetailFile(failureDetail); } return e.getExitCode().getNumericExitCode(); } finally { if (sigintHandler != null) { sigintHandler.uninstall(); } } } /** * Parses the command line arguments into a {@link OptionsParser} object. * *

This function needs to parse the --option_sources option manually so that the real option * parser can set the source for every option correctly. If that cannot be parsed or is missing, * we just report an unknown source for every startup option. */ private static OptionsParsingResult parseStartupOptions( Iterable modules, List args) throws OptionsParsingException { ImmutableList> optionClasses = BlazeCommandUtils.getStartupOptions(modules); // First parse the command line so that we get the option_sources argument OptionsParser parser = OptionsParser.builder().optionsClasses(optionClasses).allowResidue(false).build(); parser.parse(PriorityCategory.COMMAND_LINE, null, args); Map optionSources = parser.getOptions(BlazeServerStartupOptions.class).optionSources; Function sourceFunction = option -> !optionSources.containsKey(option.getOptionName()) ? "default" : optionSources.get(option.getOptionName()).isEmpty() ? "command line" : optionSources.get(option.getOptionName()); // Then parse the command line again, this time with the correct option sources parser = OptionsParser.builder().optionsClasses(optionClasses).allowResidue(false).build(); parser.parseWithSourceFunction( PriorityCategory.COMMAND_LINE, sourceFunction, args, /* fallbackData= */ null); return parser; } /** * Creates a new blaze runtime, given the install and output base directories. * *

Note: This method can and should only be called once per startup, as it also creates the * filesystem object that will be used for the runtime. So it should only ever be called from the * main method of the Blaze program. * * @param args Blaze startup options. * @return a new BlazeRuntime instance initialized with the given filesystem and directories, and * an error string that, if not null, describes a fatal initialization failure that makes this * runtime unsuitable for real commands */ private static BlazeRuntime newRuntime( Iterable blazeModules, List args, Runnable abruptShutdownHandler) throws AbruptExitException, OptionsParsingException { OptionsParsingResult options = parseStartupOptions(blazeModules, args); BlazeServerStartupOptions startupOptions = options.getOptions(BlazeServerStartupOptions.class); // Set up the failure detail path first, so that it can communicate problems with other flags // and module initialization. PathFragment failureDetailOut = startupOptions.failureDetailOut; if (failureDetailOut == null || !failureDetailOut.isAbsolute()) { // (includes "" default case) throw new IllegalArgumentException( "Bad --failure_detail_out option specified: '" + failureDetailOut + "'"); } CustomFailureDetailPublisher.setFailureDetailFilePath(failureDetailOut.getPathString()); for (BlazeModule module : blazeModules) { module.globalInit(options); } String productName = startupOptions.productName.toLowerCase(Locale.US); PathFragment workspaceDirectory = startupOptions.workspaceDirectory; PathFragment defaultSystemJavabase = startupOptions.defaultSystemJavabase; PathFragment outputUserRoot = startupOptions.outputUserRoot; PathFragment installBase = startupOptions.installBase; PathFragment outputBase = startupOptions.outputBase; maybeForceJNIByGettingPid(installBase); // Must be before first use of JNI. // From the point of view of the Java program --install_base, --output_base, --output_user_root, // and --failure_detail_out are mandatory options, despite the comment in their declarations. if (installBase == null || !installBase.isAbsolute()) { // (includes "" default case) throw new IllegalArgumentException( "Bad --install_base option specified: '" + installBase + "'"); } if (outputUserRoot != null && !outputUserRoot.isAbsolute()) { // (includes "" default case) throw new IllegalArgumentException( "Bad --output_user_root option specified: '" + outputUserRoot + "'"); } if (outputBase != null && !outputBase.isAbsolute()) { // (includes "" default case) throw new IllegalArgumentException( "Bad --output_base option specified: '" + outputBase + "'"); } FileSystem nativeFs = null; Path execRootBasePath = null; for (BlazeModule module : blazeModules) { BlazeModule.ModuleFileSystem moduleFs = module.getFileSystem(options, outputBase.getRelative(ServerDirectories.EXECROOT)); if (moduleFs != null) { execRootBasePath = moduleFs.virtualExecRootBase(); Preconditions.checkState(nativeFs == null, "more than one module returns a file system"); nativeFs = moduleFs.fileSystem(); } } Preconditions.checkNotNull(nativeFs, "No module set the file system"); FileSystem maybeFsForBuildArtifacts = null; for (BlazeModule module : blazeModules) { FileSystem maybeFs = module.getFileSystemForBuildArtifacts(nativeFs); if (maybeFs != null) { checkState( maybeFsForBuildArtifacts == null, "more than one module returns a file system for build artifacts"); maybeFsForBuildArtifacts = maybeFs; } } FileSystem fs = MoreObjects.firstNonNull(maybeFsForBuildArtifacts, nativeFs); SubscriberExceptionHandler currentHandlerValue = null; for (BlazeModule module : blazeModules) { SubscriberExceptionHandler newHandler = module.getEventBusAndAsyncExceptionHandler(); if (newHandler != null) { Preconditions.checkState( currentHandlerValue == null, "Two handlers given. Last module: %s", module); currentHandlerValue = newHandler; } } if (currentHandlerValue == null) { if (startupOptions.fatalEventBusExceptions) { currentHandlerValue = (exception, context) -> BugReport.handleCrash(exception); } else { currentHandlerValue = (exception, context) -> { if (context == null) { BugReport.handleCrash(exception); } else { BugReport.sendBugReport( exception, ImmutableList.of(), "Failure in EventBus subscriber"); } }; } } SubscriberExceptionHandler subscriberExceptionHandler = currentHandlerValue; Thread.setDefaultUncaughtExceptionHandler( (thread, throwable) -> subscriberExceptionHandler.handleException(throwable, null)); // Set the hook used to display Starlark source lines in a stack trace. EvalException.setSourceReaderSupplier( () -> loc -> { try { // TODO(adonovan): opt: cache seen files, as the stack often repeats the same files. Path path = fs.getPath(PathFragment.create(loc.file())); List lines = FileSystemUtils.readLines(path, UTF_8); return lines.size() >= loc.line() ? lines.get(loc.line() - 1) : null; } catch (Throwable unused) { // ignore any failure (e.g. ENOENT, security manager rejecting I/O) } return null; }); Path outputUserRootPath = fs.getPath(outputUserRoot); Path installBasePath = fs.getPath(installBase); Path outputBasePath = fs.getPath(outputBase); if (execRootBasePath == null) { execRootBasePath = outputBasePath.getRelative(ServerDirectories.EXECROOT); } Path workspaceDirectoryPath = null; if (!workspaceDirectory.equals(PathFragment.EMPTY_FRAGMENT)) { workspaceDirectoryPath = nativeFs.getPath(workspaceDirectory); } Path defaultSystemJavabasePath = null; if (!defaultSystemJavabase.equals(PathFragment.EMPTY_FRAGMENT)) { defaultSystemJavabasePath = fs.getPath(defaultSystemJavabase); } ServerDirectories serverDirectories = new ServerDirectories( installBasePath, outputBasePath, outputUserRootPath, execRootBasePath, startupOptions.installMD5); Clock clock = BlazeClock.instance(); BlazeRuntime.Builder runtimeBuilder = new BlazeRuntime.Builder() .setProductName(productName) .setFileSystem(fs) .setServerDirectories(serverDirectories) .setActionKeyContext(new ActionKeyContext()) .setStartupOptionsProvider(options) .setClock(clock) .setAbruptShutdownHandler(abruptShutdownHandler) .setEventBusExceptionHandler(subscriberExceptionHandler); if (TestType.isInTest() && System.getenv("NO_CRASH_ON_LOGGING_IN_TEST") == null) { LoggingUtil.installRemoteLogger(getTestCrashLogger()); } for (BlazeModule blazeModule : blazeModules) { runtimeBuilder.addBlazeModule(blazeModule); } BlazeRuntime runtime = runtimeBuilder.build(); CustomExitCodePublisher.setAbruptExitStatusFileDir( serverDirectories.getOutputBase().getPathString()); BlazeDirectories directories = new BlazeDirectories( serverDirectories, workspaceDirectoryPath, defaultSystemJavabasePath, productName); BinTools binTools; try { binTools = BinTools.forProduction(directories); } catch (IOException e) { throw createFilesystemExitException( "Cannot enumerate embedded binaries: " + e.getMessage(), Filesystem.Code.EMBEDDED_BINARIES_ENUMERATION_FAILURE, e); } // Keep this line last in this method, so that all other initialization is available to it. runtime.initWorkspace(directories, binTools); return runtime; } private static AbruptExitException createFilesystemExitException( String message, Filesystem.Code detailedCode, Exception e) { return new AbruptExitException( DetailedExitCode.of( FailureDetail.newBuilder() .setMessage(message) .setFilesystem(Filesystem.newBuilder().setCode(detailedCode)) .build()), e); } private static String maybeGetPidString() { Integer pid = maybeForceJNIByGettingPid(null); return pid == null ? "" : "pid " + pid + " and "; } /** Loads JNI libraries, if necessary under the current platform. */ @Nullable private static Integer maybeForceJNIByGettingPid(@Nullable PathFragment installBase) { return JniLoader.isJniAvailable() ? getPidUsingJNI(installBase) : null; } // Force JNI linking at a moment when we have 'installBase' handy, and print // an informative error if it fails. private static int getPidUsingJNI(@Nullable PathFragment installBase) { try { return ProcessUtils.getpid(); // force JNI initialization } catch (UnsatisfiedLinkError t) { System.err.println( "JNI initialization failed: " + t.getMessage() + ". " + "Possibly your installation has been corrupted" + (installBase == null ? "" : "; if this problem persists, try 'rm -fr " + installBase + "'") + "."); throw t; } } /** * Returns a logger that crashes as soon as it's written to, since tests should not cause events * that would be logged. */ @VisibleForTesting public static Future getTestCrashLogger() { Logger crashLogger = Logger.getAnonymousLogger(); crashLogger.addHandler( new Handler() { @Override public void publish(LogRecord record) { System.err.println("Remote logging disabled for testing, forcing abrupt shutdown."); System.err.printf( "%s#%s: %s\n", record.getSourceClassName(), record.getSourceMethodName(), record.getMessage()); Throwable e = record.getThrown(); if (e != null) { e.printStackTrace(); } Runtime.getRuntime().halt(ExitCode.BLAZE_INTERNAL_ERROR.getNumericExitCode()); } @Override public void flush() { throw new IllegalStateException(); } @Override public void close() { throw new IllegalStateException(); } }); return Futures.immediateFuture(crashLogger); } /** * Make sure async threads cannot be orphaned at startup. This method makes sure bugs are reported * to telemetry and the proper exit code is reported. Will be overwritten with better handler. */ private static void setupUncaughtHandlerAtStartup(final String[] args) { Thread.setDefaultUncaughtExceptionHandler( (thread, throwable) -> BugReport.handleCrash(throwable, args)); } @Override public String getProductName() { return productName; } @Override public void fillInCrashContext(CrashContext ctx) { CommandEnvironment localEnv = env; if (localEnv == null) { return; } CommonCommandOptions options = localEnv.getOptions().getOptions(CommonCommandOptions.class); if (options.heapDumpOnOom) { ctx.setHeapDumpPath( workspace .getOutputBase() .getRelative(env.getCommandId() + ".heapdump.hprof") // Must end in .hprof. .getPathString()); } ctx.withExtraOomInfo(options.oomMessage).reportingTo(localEnv.getReporter()); } public BuildEventArtifactUploaderFactoryMap getBuildEventArtifactUploaderFactoryMap() { return buildEventArtifactUploaderFactoryMap; } /** Returns a map of all registered {@link AuthHeadersProvider}s. */ public ImmutableMap getAuthHeadersProvidersMap() { return authHeadersProviderMap; } public RepositoryRemoteExecutorFactory getRepositoryRemoteExecutorFactory() { return repositoryRemoteExecutorFactory; } public Supplier getDownloaderSupplier() { return downloaderSupplier; } /** * A builder for {@link BlazeRuntime} objects. The only required fields are the {@link * BlazeDirectories}, and the {@link com.google.devtools.build.lib.packages.RuleClassProvider} * (except for testing). All other fields have safe default values. * *

The default behavior of the BlazeRuntime's EventBus is to exit the JVM when a subscriber * throws an exception. Please plan appropriately. */ public static class Builder { private FileSystem fileSystem; private ServerDirectories serverDirectories; private Clock clock; private Runnable abruptShutdownHandler; private OptionsParsingResult startupOptionsProvider; private final List blazeModules = new ArrayList<>(); private SubscriberExceptionHandler eventBusExceptionHandler = (throwable, context) -> BugReport.handleCrash(throwable); private UUID instanceId; private String productName; private ActionKeyContext actionKeyContext; private BugReporter bugReporter = BugReporter.defaultInstance(); @VisibleForTesting public BlazeRuntime build() throws AbruptExitException { Preconditions.checkNotNull(productName); Preconditions.checkNotNull(serverDirectories); Preconditions.checkNotNull(startupOptionsProvider); ActionKeyContext actionKeyContext = this.actionKeyContext != null ? this.actionKeyContext : new ActionKeyContext(); Clock clock = (this.clock == null) ? BlazeClock.instance() : this.clock; UUID instanceId = (this.instanceId == null) ? UUID.randomUUID() : this.instanceId; Preconditions.checkNotNull(clock); int metricsModules = 0; for (BlazeModule module : blazeModules) { if (module.postsBuildMetricsEvent()) { metricsModules++; } } Preconditions.checkArgument( metricsModules < 2, "At most one module may post a BuildMetricsEvent"); if (metricsModules == 0) { blazeModules.add(new DummyMetricsModule()); } for (BlazeModule module : blazeModules) { module.blazeStartup( startupOptionsProvider, BlazeVersionInfo.instance(), instanceId, fileSystem, serverDirectories, clock); } ServerBuilder serverBuilder = new ServerBuilder(); serverBuilder.addQueryOutputFormatters(OutputFormatters.getDefaultFormatters()); for (BlazeModule module : blazeModules) { module.serverInit(startupOptionsProvider, serverBuilder); } ConfiguredRuleClassProvider.Builder ruleClassBuilder = new ConfiguredRuleClassProvider.Builder(); for (BlazeModule module : blazeModules) { module.initializeRuleClasses(ruleClassBuilder); } ConfiguredRuleClassProvider ruleClassProvider = ruleClassBuilder.build(); PackageSettings packageSettings = getPackageSettings(blazeModules); PackageFactory packageFactory = new PackageFactory( ruleClassProvider, PackageFactory.makeDefaultSizedForkJoinPoolForGlobbing(), packageSettings, getPackageValidator(blazeModules), getPackageOverheadEstimator(blazeModules), getPackageLoadingListener( blazeModules, packageSettings, ruleClassProvider, fileSystem)); ProjectFile.Provider projectFileProvider = null; for (BlazeModule module : blazeModules) { ProjectFile.Provider candidate = module.createProjectFileProvider(); if (candidate != null) { Preconditions.checkState( projectFileProvider == null, "more than one module defines a project file provider"); projectFileProvider = candidate; } } QueryRuntimeHelper.Factory queryRuntimeHelperFactory = null; for (BlazeModule module : blazeModules) { QueryRuntimeHelper.Factory candidateFactory = module.getQueryRuntimeHelperFactory(); if (candidateFactory != null) { Preconditions.checkState( queryRuntimeHelperFactory == null, "more than one module defines a query helper factory"); queryRuntimeHelperFactory = candidateFactory; } } if (queryRuntimeHelperFactory == null) { queryRuntimeHelperFactory = QueryRuntimeHelper.StdoutQueryRuntimeHelperFactory.INSTANCE; } BlazeRuntime runtime = new BlazeRuntime( fileSystem, serverBuilder.getQueryEnvironmentFactory(), serverBuilder.getQueryFunctions(), serverBuilder.getQueryOutputFormatters(), packageFactory, ruleClassProvider, serverBuilder.getInfoItems(), actionKeyContext, clock, abruptShutdownHandler, startupOptionsProvider, ImmutableList.copyOf(blazeModules), eventBusExceptionHandler, bugReporter, projectFileProvider, queryRuntimeHelperFactory, serverBuilder.getInvocationPolicy(), serverBuilder.getCommands(), productName, serverBuilder.getBuildEventArtifactUploaderMap(), serverBuilder.getAuthHeadersProvidersMap(), serverBuilder.getRepositoryRemoteExecutorFactory(), serverBuilder.getDownloaderSupplier()); AutoProfiler.setClock(runtime.getClock()); BugReport.setRuntime(runtime); return runtime; } @CanIgnoreReturnValue public Builder setProductName(String productName) { this.productName = productName; return this; } @CanIgnoreReturnValue public Builder setFileSystem(FileSystem fileSystem) { this.fileSystem = fileSystem; return this; } @CanIgnoreReturnValue public Builder setServerDirectories(ServerDirectories serverDirectories) { this.serverDirectories = serverDirectories; return this; } @CanIgnoreReturnValue public Builder setClock(Clock clock) { this.clock = clock; return this; } @CanIgnoreReturnValue public Builder setAbruptShutdownHandler(Runnable handler) { this.abruptShutdownHandler = handler; return this; } @CanIgnoreReturnValue public Builder setStartupOptionsProvider(OptionsParsingResult startupOptionsProvider) { this.startupOptionsProvider = startupOptionsProvider; return this; } @CanIgnoreReturnValue public Builder addBlazeModule(BlazeModule blazeModule) { blazeModules.add(blazeModule); return this; } @CanIgnoreReturnValue public Builder setInstanceId(UUID id) { instanceId = id; return this; } @CanIgnoreReturnValue @VisibleForTesting public Builder setEventBusExceptionHandler( SubscriberExceptionHandler eventBusExceptionHandler) { this.eventBusExceptionHandler = eventBusExceptionHandler; return this; } @CanIgnoreReturnValue @VisibleForTesting public Builder setBugReporter(BugReporter bugReporter) { this.bugReporter = bugReporter; return this; } @CanIgnoreReturnValue public Builder setActionKeyContext(ActionKeyContext actionKeyContext) { this.actionKeyContext = actionKeyContext; return this; } private static PackageSettings getPackageSettings(List blazeModules) { List packageSettingss = blazeModules.stream() .map(BlazeModule::getPackageSettings) .filter(Objects::nonNull) .collect(toImmutableList()); Preconditions.checkState( packageSettingss.size() <= 1, "more than one module defines a PackageSettings"); return Iterables.getFirst(packageSettingss, PackageSettings.DEFAULTS); } private static PackageValidator getPackageValidator(List blazeModules) { List packageValidators = blazeModules.stream() .map(BlazeModule::getPackageValidator) .filter(Objects::nonNull) .collect(toImmutableList()); Preconditions.checkState( packageValidators.size() <= 1, "more than one module defined a PackageValidator"); return Iterables.getFirst(packageValidators, PackageValidator.NOOP_VALIDATOR); } private static PackageOverheadEstimator getPackageOverheadEstimator( List blazeModules) { List packageOverheadEstimators = blazeModules.stream() .map(BlazeModule::getPackageOverheadEstimator) .filter(Objects::nonNull) .collect(toImmutableList()); Preconditions.checkState( packageOverheadEstimators.size() <= 1, "more than one module defined a PackageOverheadEstimator"); return Iterables.getFirst(packageOverheadEstimators, PackageOverheadEstimator.NOOP_ESTIMATOR); } } private static PackageLoadingListener getPackageLoadingListener( List blazeModules, PackageSettings packageBuilderHelper, ConfiguredRuleClassProvider ruleClassProvider, FileSystem fs) { ImmutableList listeners = blazeModules.stream() .map( module -> module.getPackageLoadingListener(packageBuilderHelper, ruleClassProvider, fs)) .filter(Objects::nonNull) .collect(toImmutableList()); return PackageLoadingListener.create(listeners); } private static int readPidFile(Path pidFile) throws AbruptExitException { try { return Integer.parseInt(new String(FileSystemUtils.readContentAsLatin1(pidFile))); } catch (IOException e) { throw createFilesystemExitException( "Server pid file read failed: " + e.getMessage(), Filesystem.Code.SERVER_PID_TXT_FILE_READ_FAILURE, e); } catch (NumberFormatException e) { // Invalid contents (not a number) is more likely than not a filesystem issue. throw createFilesystemExitException( "Server pid file corrupted: " + e.getMessage(), Filesystem.Code.SERVER_PID_TXT_FILE_READ_FAILURE, new IOException(e)); } } }