// Copyright 2018 The Bazel Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package com.google.devtools.build.lib.bazel.commands; import static com.google.common.collect.ImmutableList.toImmutableList; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.devtools.build.lib.analysis.NoBuildEvent; import com.google.devtools.build.lib.analysis.NoBuildRequestFinishedEvent; import com.google.devtools.build.lib.bazel.ResolvedEvent; import com.google.devtools.build.lib.bazel.repository.RepositoryOrderEvent; import com.google.devtools.build.lib.bazel.repository.starlark.StarlarkRepositoryFunction; import com.google.devtools.build.lib.cmdline.Label; import com.google.devtools.build.lib.cmdline.LabelConstants; import com.google.devtools.build.lib.cmdline.LabelSyntaxException; import com.google.devtools.build.lib.cmdline.RepositoryName; import com.google.devtools.build.lib.cmdline.TargetPattern; import com.google.devtools.build.lib.events.Event; import com.google.devtools.build.lib.packages.Rule; import com.google.devtools.build.lib.packages.WorkspaceFileValue; import com.google.devtools.build.lib.pkgcache.PackageOptions; import com.google.devtools.build.lib.rules.repository.RepositoryDelegatorFunction; import com.google.devtools.build.lib.rules.repository.RepositoryDirectoryValue; import com.google.devtools.build.lib.rules.repository.ResolvedHashesFunction; import com.google.devtools.build.lib.runtime.BlazeCommand; import com.google.devtools.build.lib.runtime.BlazeCommandResult; import com.google.devtools.build.lib.runtime.Command; import com.google.devtools.build.lib.runtime.CommandEnvironment; import com.google.devtools.build.lib.runtime.KeepGoingOption; import com.google.devtools.build.lib.runtime.LoadingPhaseThreadsOption; import com.google.devtools.build.lib.server.FailureDetails; import com.google.devtools.build.lib.server.FailureDetails.FailureDetail; import com.google.devtools.build.lib.server.FailureDetails.SyncCommand.Code; import com.google.devtools.build.lib.skyframe.PackageLookupValue; import com.google.devtools.build.lib.skyframe.PrecomputedValue; import com.google.devtools.build.lib.skyframe.SkyframeExecutor; import com.google.devtools.build.lib.util.AbruptExitException; import com.google.devtools.build.lib.util.DetailedExitCode; import com.google.devtools.build.lib.util.ExitCode; import com.google.devtools.build.lib.util.InterruptedFailureDetails; import com.google.devtools.build.lib.vfs.RootedPath; import com.google.devtools.build.lib.vfs.XattrProvider; import com.google.devtools.build.skyframe.EvaluationContext; import com.google.devtools.build.skyframe.EvaluationResult; import com.google.devtools.build.skyframe.SkyKey; import com.google.devtools.build.skyframe.SkyValue; import com.google.devtools.common.options.OptionsParsingResult; import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.stream.Collectors; import net.starlark.java.eval.Starlark; /** Syncs all repositories specified in the workspace file */ @Command( name = SyncCommand.NAME, options = { PackageOptions.class, KeepGoingOption.class, LoadingPhaseThreadsOption.class, SyncOptions.class }, help = "resource:sync.txt", shortDescription = "Syncs all repositories specified in the workspace file", allowResidue = false) public final class SyncCommand implements BlazeCommand { public static final String NAME = "sync"; static final ImmutableSet WHITELISTED_NATIVE_RULES = ImmutableSet.of("local_repository", "new_local_repository", "local_config_platform"); private static void reportError(CommandEnvironment env, EvaluationResult value) { if (value.getError().getException() != null) { env.getReporter().handle(Event.error(value.getError().getException().getMessage())); } else { env.getReporter().handle(Event.error(value.getError().toString())); } } @Override public BlazeCommandResult exec(CommandEnvironment env, OptionsParsingResult options) { try { env.getReporter() .post( new NoBuildEvent( env.getCommandName(), env.getCommandStartTime(), true, true, env.getCommandId().toString())); env.syncPackageLoading(options); SkyframeExecutor skyframeExecutor = env.getSkyframeExecutor(); SyncOptions syncOptions = options.getOptions(SyncOptions.class); if (syncOptions.configure) { skyframeExecutor.injectExtraPrecomputedValues( ImmutableList.of( PrecomputedValue.injected( RepositoryDelegatorFunction.DEPENDENCY_FOR_UNCONDITIONAL_CONFIGURING, env.getCommandId().toString()))); } else { skyframeExecutor.injectExtraPrecomputedValues( ImmutableList.of( PrecomputedValue.injected( RepositoryDelegatorFunction.DEPENDENCY_FOR_UNCONDITIONAL_FETCHING, env.getCommandId().toString()))); } // Obtain the key for the top-level WORKSPACE file SkyKey packageLookupKey = PackageLookupValue.key(LabelConstants.EXTERNAL_PACKAGE_IDENTIFIER); LoadingPhaseThreadsOption threadsOption = options.getOptions(LoadingPhaseThreadsOption.class); EvaluationContext evaluationContext = EvaluationContext.newBuilder() .setParallelism(threadsOption.threads) .setEventHandler(env.getReporter()) .build(); EvaluationResult packageLookupValue = skyframeExecutor.prepareAndGet(ImmutableSet.of(packageLookupKey), evaluationContext); if (packageLookupValue.hasError()) { reportError(env, packageLookupValue); return blazeCommandResultWithNoBuildReport( env, ExitCode.ANALYSIS_FAILURE, Code.PACKAGE_LOOKUP_ERROR, packageLookupValue.getError(packageLookupKey).toString()); } RootedPath workspacePath = ((PackageLookupValue) packageLookupValue.get(packageLookupKey)) .getRootedPath(LabelConstants.EXTERNAL_PACKAGE_IDENTIFIER); SkyKey workspace = WorkspaceFileValue.key(workspacePath); // read and evaluate the WORKSPACE file to its end ImmutableList.Builder repositoryOrder = new ImmutableList.Builder<>(); Set namesSeen = new HashSet<>(); WorkspaceFileValue fileValue = null; while (workspace != null) { EvaluationResult value = skyframeExecutor.prepareAndGet(ImmutableSet.of(workspace), evaluationContext); if (value.hasError()) { reportError(env, value); return blazeCommandResultWithNoBuildReport( env, ExitCode.ANALYSIS_FAILURE, Code.WORKSPACE_EVALUATION_ERROR, value.getError(workspace).toString()); } fileValue = (WorkspaceFileValue) value.get(workspace); for (Rule rule : fileValue.getPackage().getTargets(Rule.class)) { String name = rule.getName(); if (!namesSeen.contains(name)) { repositoryOrder.add(name); namesSeen.add(name); } } workspace = fileValue.next(); } env.getReporter() .post( genericArgsCall( "register_toolchains", fileValue.getPackage().getRegisteredToolchains().stream() .map(TargetPattern::getOriginalPattern) .collect(toImmutableList()))); env.getReporter() .post( genericArgsCall( "register_execution_platforms", fileValue.getPackage().getRegisteredExecutionPlatforms().stream() .map(TargetPattern::getOriginalPattern) .collect(toImmutableList()))); env.getReporter().post(new RepositoryOrderEvent(repositoryOrder.build())); // take all Starlark workspace rules and get their values ImmutableSet.Builder repositoriesToFetch = new ImmutableSet.Builder<>(); for (Rule rule : fileValue.getPackage().getTargets(Rule.class)) { if (rule.getRuleClass().equals("bind")) { // The bind rule is special in that the name is not that of an external repository. // Moreover, it is not affected by the invalidation mechanism as there is nothing to // fetch anyway. So the only task remaining is to record the use of "bind" for whoever // collects resolved information. env.getReporter().post(resolveBind(rule)); } else if (shouldSync(rule, syncOptions)) { // TODO(aehlig): avoid the detour of serializing and then parsing the repository name try { repositoriesToFetch.add( RepositoryDirectoryValue.key(RepositoryName.create(rule.getName()))); } catch (LabelSyntaxException e) { String errorMessage = String.format( "Internal error queuing %s to fetch: %s", rule.getName(), e.getMessage()); env.getReporter().handle(Event.error(errorMessage)); return blazeCommandResultWithNoBuildReport( env, ExitCode.BLAZE_INTERNAL_ERROR, Code.REPOSITORY_NAME_INVALID, errorMessage); } } } EvaluationResult fetchValue; fetchValue = skyframeExecutor.prepareAndGet(repositoriesToFetch.build(), evaluationContext); if (fetchValue.hasError()) { reportError(env, fetchValue); return blazeCommandResultWithNoBuildReport( env, ExitCode.ANALYSIS_FAILURE, Code.REPOSITORY_FETCH_ERRORS, "Repository fetch failure."); } } catch (InterruptedException e) { String errorMessage = "Sync interrupted: " + e.getMessage(); env.getReporter().handle(Event.error(errorMessage)); reportNoBuildRequestFinished(env, ExitCode.INTERRUPTED); return BlazeCommandResult.detailedExitCode( InterruptedFailureDetails.detailedExitCode(errorMessage)); } catch (AbruptExitException e) { env.getReporter().handle(Event.error("Unknown error: " + e.getMessage())); reportNoBuildRequestFinished(env, ExitCode.LOCAL_ENVIRONMENTAL_ERROR); return BlazeCommandResult.detailedExitCode(e.getDetailedExitCode()); } reportNoBuildRequestFinished(env, ExitCode.SUCCESS); return BlazeCommandResult.success(); } private static boolean shouldSync(Rule rule, SyncOptions options) { if (!rule.getRuleClassObject().getWorkspaceOnly()) { // We should only sync workspace rules return false; } if (options.only != null && !options.only.isEmpty() && !options.only.contains(rule.getName())) { // There is a whitelist of what to sync, but the rule is not in this white list return false; } if (options.configure) { // If this is only a configure run, only sync Starlark rules that // declare themselves as configure-like. return StarlarkRepositoryFunction.isConfigureRule(rule); } if (rule.getRuleClassObject().isStarlark()) { // Starlark rules are all whitelisted return true; } return WHITELISTED_NATIVE_RULES.contains(rule.getRuleClassObject().getName()); } private static ResolvedEvent resolveBind(Rule rule) { String name = rule.getName(); Label actual = (Label) rule.getAttr("actual"); String nativeCommand = String.format( "bind(name = %s, actual = %s)", Starlark.repr(name), Starlark.repr(actual.getCanonicalForm())); return new ResolvedEvent() { @Override public String getName() { return name; } @Override public Object getResolvedInformation(XattrProvider xattrProvider) { return ImmutableMap.builder() .put(ResolvedHashesFunction.ORIGINAL_RULE_CLASS, "bind") .put( ResolvedHashesFunction.ORIGINAL_ATTRIBUTES, ImmutableMap.of("name", name, "actual", actual)) .put(ResolvedHashesFunction.NATIVE, nativeCommand) .buildOrThrow(); } }; } private static ResolvedEvent genericArgsCall(String ruleName, List args) { // For the name attribute we are in a slightly tricky situation, as the ResolvedEvents are // designed for external repositories and hence are indexted by their unique // names. Technically, however, things like the list of toolchains are not associated with any // external repository (but still a workspace command); so we take a name that syntactially can // never be the name of a repository, as it starts with a '//'. String name = "//external/" + ruleName; StringBuilder nativeCommandBuilder = new StringBuilder().append(ruleName).append("("); nativeCommandBuilder.append( args.stream().map(Starlark::repr).collect(Collectors.joining(", "))); nativeCommandBuilder.append(")"); String nativeCommand = nativeCommandBuilder.toString(); return new ResolvedEvent() { @Override public String getName() { return name; } @Override public Object getResolvedInformation(XattrProvider xattrProvider) { return ImmutableMap.builder() .put(ResolvedHashesFunction.ORIGINAL_RULE_CLASS, ruleName) .put( ResolvedHashesFunction.ORIGINAL_ATTRIBUTES, // The original attributes are a bit of a problem, as the arguments to // the rule do not at all look like those of a repository rule: // they're all positional, and, in particular, there is no keyword argument // called "name". A lot of uses of the resolved file, however, blindly assume // that "name" is always part of the original arguments; so we provide our // fake name here as well, and the actual arguments under the keyword "*args", // which hopefully reminds everyone inspecting the file of the actual syntax of // that rule. Note that the original arguments are always ignored when bazel uses // a resolved file instead of a workspace file. ImmutableMap.of("name", name, "*args", args)) .put(ResolvedHashesFunction.NATIVE, nativeCommand) .build(); } }; } private static BlazeCommandResult blazeCommandResultWithNoBuildReport( CommandEnvironment env, ExitCode exitCode, Code syncCommandCode, String message) { reportNoBuildRequestFinished(env, exitCode); return createFailedBlazeCommandResult(syncCommandCode, message); } private static void reportNoBuildRequestFinished(CommandEnvironment env, ExitCode exitCode) { long finishTimeMillis = env.getRuntime().getClock().currentTimeMillis(); env.getReporter().post(new NoBuildRequestFinishedEvent(exitCode, finishTimeMillis)); } private static BlazeCommandResult createFailedBlazeCommandResult( Code syncCommandCode, String message) { return BlazeCommandResult.detailedExitCode( DetailedExitCode.of( FailureDetail.newBuilder() .setMessage(message) .setSyncCommand( FailureDetails.SyncCommand.newBuilder().setCode(syncCommandCode).build()) .build())); } }