// Copyright 2017 The Bazel Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package com.google.devtools.build.lib.remote; import static com.google.common.truth.Truth.assertThat; import static com.google.common.util.concurrent.MoreExecutors.directExecutor; import static java.nio.charset.StandardCharsets.ISO_8859_1; import static org.junit.Assert.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.inOrder; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; import build.bazel.remote.execution.v2.ActionResult; import build.bazel.remote.execution.v2.Digest; import build.bazel.remote.execution.v2.ExecuteOperationMetadata; import build.bazel.remote.execution.v2.ExecuteRequest; import build.bazel.remote.execution.v2.ExecuteResponse; import build.bazel.remote.execution.v2.ExecutedActionMetadata; import build.bazel.remote.execution.v2.ExecutionStage.Value; import build.bazel.remote.execution.v2.LogFile; import com.google.common.collect.ClassToInstanceMap; import com.google.common.collect.ImmutableClassToInstanceMap; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.eventbus.EventBus; import com.google.common.io.ByteStreams; import com.google.common.util.concurrent.ListeningScheduledExecutorService; import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.SettableFuture; import com.google.devtools.build.lib.actions.ActionContext; import com.google.devtools.build.lib.actions.ActionInput; import com.google.devtools.build.lib.actions.ActionInputMap; import com.google.devtools.build.lib.actions.Artifact; import com.google.devtools.build.lib.actions.ArtifactRoot; import com.google.devtools.build.lib.actions.ArtifactRoot.RootType; import com.google.devtools.build.lib.actions.CommandLines.ParamFileActionInput; import com.google.devtools.build.lib.actions.ExecutionRequirements; import com.google.devtools.build.lib.actions.ParameterFile.ParameterFileType; import com.google.devtools.build.lib.actions.ResourceSet; import com.google.devtools.build.lib.actions.SimpleSpawn; import com.google.devtools.build.lib.actions.Spawn; import com.google.devtools.build.lib.actions.SpawnMetrics; import com.google.devtools.build.lib.actions.SpawnResult; import com.google.devtools.build.lib.actions.SpawnResult.Status; import com.google.devtools.build.lib.actions.StaticInputMetadataProvider; import com.google.devtools.build.lib.clock.JavaClock; import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; import com.google.devtools.build.lib.collect.nestedset.Order; import com.google.devtools.build.lib.events.Reporter; import com.google.devtools.build.lib.exec.AbstractSpawnStrategy; import com.google.devtools.build.lib.exec.ExecutionOptions; import com.google.devtools.build.lib.exec.RemoteLocalFallbackRegistry; import com.google.devtools.build.lib.exec.SpawnCheckingCacheEvent; import com.google.devtools.build.lib.exec.SpawnExecutingEvent; import com.google.devtools.build.lib.exec.SpawnRunner; import com.google.devtools.build.lib.exec.SpawnRunner.SpawnExecutionContext; import com.google.devtools.build.lib.exec.SpawnSchedulingEvent; import com.google.devtools.build.lib.exec.util.FakeOwner; import com.google.devtools.build.lib.remote.RemoteExecutionService.RemoteActionResult; import com.google.devtools.build.lib.remote.common.BulkTransferException; import com.google.devtools.build.lib.remote.common.CacheNotFoundException; import com.google.devtools.build.lib.remote.common.OperationObserver; import com.google.devtools.build.lib.remote.common.RemoteActionExecutionContext; import com.google.devtools.build.lib.remote.common.RemoteCacheClient.ActionKey; import com.google.devtools.build.lib.remote.common.RemoteCacheClient.CachedActionResult; import com.google.devtools.build.lib.remote.common.RemoteExecutionClient; import com.google.devtools.build.lib.remote.common.RemotePathResolver; import com.google.devtools.build.lib.remote.common.RemotePathResolver.SiblingRepositoryLayoutResolver; import com.google.devtools.build.lib.remote.options.RemoteOptions; import com.google.devtools.build.lib.remote.options.RemoteOutputsMode; import com.google.devtools.build.lib.remote.util.DigestUtil; import com.google.devtools.build.lib.remote.util.FakeSpawnExecutionContext; import com.google.devtools.build.lib.remote.util.TempPathGenerator; import com.google.devtools.build.lib.testutil.TestUtils; import com.google.devtools.build.lib.util.ExitCode; import com.google.devtools.build.lib.util.io.FileOutErr; import com.google.devtools.build.lib.vfs.DigestHashFunction; import com.google.devtools.build.lib.vfs.FileSystem; import com.google.devtools.build.lib.vfs.OutputPermissions; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; import com.google.devtools.build.lib.vfs.SyscallCache; import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem; import com.google.devtools.common.options.Options; import com.google.longrunning.Operation; import com.google.protobuf.Any; import com.google.protobuf.Timestamp; import com.google.protobuf.util.Durations; import com.google.protobuf.util.Timestamps; import com.google.rpc.Code; import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.time.Duration; import java.time.Instant; import java.util.List; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import javax.annotation.Nullable; import org.junit.After; import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; import org.mockito.ArgumentCaptor; import org.mockito.InOrder; import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.MockitoAnnotations; import org.mockito.junit.MockitoJUnit; import org.mockito.junit.MockitoRule; /** Tests for {@link com.google.devtools.build.lib.remote.RemoteSpawnRunner} */ @RunWith(JUnit4.class) public class RemoteSpawnRunnerTest { @Rule public final MockitoRule mockito = MockitoJUnit.rule(); @Mock private RemoteOutputChecker remoteOutputChecker; // download nothing by default. private final Reporter reporter = new Reporter(new EventBus()); private static final ImmutableMap NO_CACHE = ImmutableMap.of(ExecutionRequirements.NO_CACHE, ""); private ListeningScheduledExecutorService retryService; private FileSystem fs; private Path execRoot; private ArtifactRoot artifactRoot; private TempPathGenerator tempPathGenerator; private Path logDir; private DigestUtil digestUtil; private FakeActionInputFileCache fakeFileCache; private FileOutErr outErr; private RemoteOptions remoteOptions; @Mock private RemoteExecutionCache cache; @Mock private RemoteExecutionClient executor; @Mock private SpawnRunner localRunner; // The action key of the Spawn returned by newSimpleSpawn(). private static final String SIMPLE_ACTION_ID = "31aea267dc597b047a9b6993100415b6406f82822318dc8988e4164a535b51ee"; @Before public final void setUp() throws Exception { MockitoAnnotations.initMocks(this); digestUtil = new DigestUtil(SyscallCache.NO_CACHE, DigestHashFunction.SHA256); fs = new InMemoryFileSystem(new JavaClock(), DigestHashFunction.SHA256); execRoot = fs.getPath("/exec/root"); execRoot.createDirectoryAndParents(); artifactRoot = ArtifactRoot.asDerivedRoot(execRoot, RootType.Output, "outputs"); artifactRoot.getRoot().asPath().createDirectoryAndParents(); tempPathGenerator = new TempPathGenerator(fs.getPath("/execroot/_tmp/actions/remote")); logDir = fs.getPath("/server-logs"); fakeFileCache = new FakeActionInputFileCache(execRoot); Path stdout = fs.getPath("/tmp/stdout"); Path stderr = fs.getPath("/tmp/stderr"); stdout.getParentDirectory().createDirectoryAndParents(); stderr.getParentDirectory().createDirectoryAndParents(); outErr = new FileOutErr(stdout, stderr); remoteOptions = Options.getDefaults(RemoteOptions.class); retryService = MoreExecutors.listeningDecorator(Executors.newScheduledThreadPool(1)); } @After public void afterEverything() throws InterruptedException { retryService.shutdownNow(); retryService.awaitTermination(TestUtils.WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); } @Test public void nonCachableSpawnsShouldNotBeCached_remote() throws Exception { // Test that if a spawn is marked "NO_CACHE" then it's not fetched from a remote cache. // It should be executed remotely, but marked non-cacheable to remote execution, so that // the action result is not saved in the remote cache. remoteOptions.remoteAcceptCached = true; remoteOptions.remoteLocalFallback = false; remoteOptions.remoteUploadLocalResults = true; remoteOptions.remoteResultCachePriority = 1; remoteOptions.remoteExecutionPriority = 2; RemoteSpawnRunner runner = newSpawnRunner(); RemoteExecutionService service = runner.getRemoteExecutionService(); ExecuteResponse succeeded = ExecuteResponse.newBuilder() .setResult(ActionResult.newBuilder().setExitCode(0).build()) .build(); when(executor.executeRemotely( any(RemoteActionExecutionContext.class), any(ExecuteRequest.class), any(OperationObserver.class))) .thenReturn(succeeded); Spawn spawn = simpleSpawnWithExecutionInfo(NO_CACHE); SpawnExecutionContext policy = getSpawnContext(spawn); runner.exec(spawn, policy); ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(ExecuteRequest.class); verify(executor) .executeRemotely( any(RemoteActionExecutionContext.class), requestCaptor.capture(), any(OperationObserver.class)); assertThat(requestCaptor.getValue().getSkipCacheLookup()).isTrue(); assertThat(requestCaptor.getValue().getResultsCachePolicy().getPriority()).isEqualTo(1); assertThat(requestCaptor.getValue().getExecutionPolicy().getPriority()).isEqualTo(2); // TODO(olaola): verify that the uploaded action has the doNotCache set. verify(service, never()).lookupCache(any()); verify(service, never()).uploadOutputs(any(), any()); verifyNoMoreInteractions(localRunner); } private FakeSpawnExecutionContext getSpawnContext(Spawn spawn) { AbstractSpawnStrategy fakeLocalStrategy = new AbstractSpawnStrategy(execRoot, localRunner, new ExecutionOptions()) {}; ClassToInstanceMap actionContextRegistry = ImmutableClassToInstanceMap.of(RemoteLocalFallbackRegistry.class, () -> fakeLocalStrategy); var actionInputFetcher = new RemoteActionInputFetcher( new Reporter(new EventBus()), "none", "none", cache, execRoot, tempPathGenerator, remoteOutputChecker, OutputPermissions.READONLY); var actionFileSystem = new RemoteActionFileSystem( fs, execRoot.asFragment(), artifactRoot.getRoot().asPath().relativeTo(execRoot).getPathString(), new ActionInputMap(0), ImmutableList.of(), StaticInputMetadataProvider.empty(), actionInputFetcher); return new FakeSpawnExecutionContext( spawn, fakeFileCache, execRoot, outErr, actionContextRegistry, actionFileSystem); } @Test public void nonCachableSpawnsShouldNotBeCached_localFallback() throws Exception { // Test that if a non-cachable spawn is executed locally due to the local fallback, // that its result is not uploaded to the remote cache. remoteOptions.remoteAcceptCached = true; remoteOptions.remoteLocalFallback = true; remoteOptions.remoteUploadLocalResults = true; RemoteSpawnRunner runner = newSpawnRunner(); // Throw an IOException to trigger the local fallback. when(executor.executeRemotely( any(RemoteActionExecutionContext.class), any(ExecuteRequest.class), any(OperationObserver.class))) .thenThrow(IOException.class); Spawn spawn = simpleSpawnWithExecutionInfo(NO_CACHE); SpawnExecutionContext policy = getSpawnContext(spawn); runner.exec(spawn, policy); verify(localRunner).exec(spawn, policy); verify(cache).ensureInputsPresent(any(), any(), any(), anyBoolean()); verifyNoMoreInteractions(cache); } @Test public void cachableSpawnsShouldBeCached_localFallback() throws Exception { // Test that if a cacheable spawn is executed locally due to the local fallback, // that its result is uploaded to the remote cache. remoteOptions.remoteAcceptCached = true; remoteOptions.remoteLocalFallback = true; remoteOptions.remoteUploadLocalResults = true; RemoteSpawnRunner runner = spy(newSpawnRunner()); RemoteExecutionService service = runner.getRemoteExecutionService(); doNothing().when(service).uploadOutputs(any(), any()); // Throw an IOException to trigger the local fallback. when(executor.executeRemotely( any(RemoteActionExecutionContext.class), any(ExecuteRequest.class), any(OperationObserver.class))) .thenThrow(IOException.class); SpawnResult res = new SpawnResult.Builder() .setStatus(Status.SUCCESS) .setExitCode(0) .setRunnerName("test") .build(); when(localRunner.exec(any(Spawn.class), any(SpawnExecutionContext.class))).thenReturn(res); Spawn spawn = newSimpleSpawn(); SpawnExecutionContext policy = getSpawnContext(spawn); SpawnResult result = runner.exec(spawn, policy); assertThat(result.exitCode()).isEqualTo(0); assertThat(result.status()).isEqualTo(Status.SUCCESS); verify(localRunner).exec(eq(spawn), eq(policy)); verify(runner) .execLocallyAndUpload(any(), eq(spawn), eq(policy), /* uploadLocalResults= */ eq(true)); verify(service).uploadOutputs(any(), eq(res)); } @Test public void failedLocalActionShouldNotBeUploaded() throws Exception { // Test that the outputs of a locally executed action that failed are not uploaded. remoteOptions.remoteLocalFallback = true; remoteOptions.remoteUploadLocalResults = true; RemoteSpawnRunner runner = spy(newSpawnRunner()); RemoteExecutionService service = runner.getRemoteExecutionService(); // Throw an IOException to trigger the local fallback. when(executor.executeRemotely( any(RemoteActionExecutionContext.class), any(ExecuteRequest.class), any(OperationObserver.class))) .thenThrow(IOException.class); Spawn spawn = newSimpleSpawn(); SpawnExecutionContext policy = getSpawnContext(spawn); SpawnResult res = Mockito.mock(SpawnResult.class); when(res.exitCode()).thenReturn(1); when(res.status()).thenReturn(Status.EXECUTION_FAILED); when(localRunner.exec(eq(spawn), eq(policy))).thenReturn(res); assertThat(runner.exec(spawn, policy)).isSameInstanceAs(res); verify(localRunner).exec(eq(spawn), eq(policy)); verify(runner) .execLocallyAndUpload(any(), eq(spawn), eq(policy), /* uploadLocalResults= */ eq(true)); verify(service, never()).uploadOutputs(any(), any()); } @Test public void treatFailedCachedActionAsCacheMiss_local() throws Exception { // Test that bazel treats failed cache action as a cache miss and attempts to execute action // locally remoteOptions.remoteLocalFallback = true; remoteOptions.remoteUploadLocalResults = true; CachedActionResult failedAction = CachedActionResult.remote(ActionResult.newBuilder().setExitCode(1).build()); when(cache.downloadActionResult( any(RemoteActionExecutionContext.class), any(ActionKey.class), /* inlineOutErr= */ eq(false))) .thenReturn(failedAction); RemoteSpawnRunner runner = spy(newSpawnRunner()); RemoteExecutionService service = runner.getRemoteExecutionService(); // Throw an IOException to trigger the local fallback. when(executor.executeRemotely( any(RemoteActionExecutionContext.class), any(ExecuteRequest.class), any(OperationObserver.class))) .thenThrow(IOException.class); doNothing().when(service).uploadOutputs(any(), any()); Spawn spawn = newSimpleSpawn(); SpawnExecutionContext policy = getSpawnContext(spawn); SpawnResult succeeded = new SpawnResult.Builder() .setStatus(Status.SUCCESS) .setExitCode(0) .setRunnerName("test") .build(); when(localRunner.exec(eq(spawn), eq(policy))).thenReturn(succeeded); SpawnResult result = runner.exec(spawn, policy); verify(localRunner).exec(eq(spawn), eq(policy)); verify(runner) .execLocallyAndUpload(any(), eq(spawn), eq(policy), /* uploadLocalResults= */ eq(true)); verify(service).uploadOutputs(any(), eq(result)); verify(service, never()).downloadOutputs(any(), any()); } @Test public void treatFailedCachedActionAsCacheMiss_remote() throws Exception { // Test that bazel treats failed cache action as a cache miss and attempts to execute action // remotely CachedActionResult failedAction = CachedActionResult.remote(ActionResult.newBuilder().setExitCode(1).build()); when(cache.downloadActionResult( any(RemoteActionExecutionContext.class), any(ActionKey.class), /* inlineOutErr= */ eq(false))) .thenReturn(failedAction); RemoteSpawnRunner runner = newSpawnRunner(); RemoteExecutionService service = runner.getRemoteExecutionService(); ExecuteResponse succeeded = ExecuteResponse.newBuilder() .setResult(ActionResult.newBuilder().setExitCode(0).build()) .build(); when(executor.executeRemotely( any(RemoteActionExecutionContext.class), any(ExecuteRequest.class), any(OperationObserver.class))) .thenReturn(succeeded); Spawn spawn = newSimpleSpawn(); SpawnExecutionContext policy = getSpawnContext(spawn); runner.exec(spawn, policy); verify(service).executeRemotely(any(), eq(false), any()); } @Test public void noRemoteExecutorFallbackFails() throws Exception { // Errors from the fallback runner should be propagated out of the remote runner. remoteOptions.remoteUploadLocalResults = true; remoteOptions.remoteLocalFallback = true; RemoteSpawnRunner runner = newSpawnRunner(); // Trigger local fallback when(executor.executeRemotely( any(RemoteActionExecutionContext.class), any(ExecuteRequest.class), any(OperationObserver.class))) .thenThrow(new IOException()); Spawn spawn = newSimpleSpawn(); SpawnExecutionContext policy = getSpawnContext(spawn); when(cache.downloadActionResult( any(RemoteActionExecutionContext.class), any(ActionKey.class), /* inlineOutErr= */ eq(false))) .thenReturn(null); IOException err = new IOException("local execution error"); when(localRunner.exec(eq(spawn), eq(policy))).thenThrow(err); IOException e = assertThrows(IOException.class, () -> runner.exec(spawn, policy)); assertThat(e).isSameInstanceAs(err); verify(localRunner).exec(eq(spawn), eq(policy)); } @Test public void remoteCacheErrorFallbackFails() throws Exception { // Errors from the fallback runner should be propagated out of the remote runner. remoteOptions.remoteUploadLocalResults = true; remoteOptions.remoteLocalFallback = true; RemoteSpawnRunner runner = newSpawnRunner(); // Trigger local fallback when(executor.executeRemotely( any(RemoteActionExecutionContext.class), any(ExecuteRequest.class), any(OperationObserver.class))) .thenThrow(new IOException()); Spawn spawn = newSimpleSpawn(); SpawnExecutionContext policy = getSpawnContext(spawn); when(cache.downloadActionResult( any(RemoteActionExecutionContext.class), any(ActionKey.class), /* inlineOutErr= */ eq(false))) .thenThrow(new IOException()); IOException err = new IOException("local execution error"); when(localRunner.exec(eq(spawn), eq(policy))).thenThrow(err); IOException e = assertThrows(IOException.class, () -> runner.exec(spawn, policy)); assertThat(e).isSameInstanceAs(err); verify(localRunner).exec(eq(spawn), eq(policy)); } @Test public void testLocalFallbackFailureRemoteExecutorFailure() throws Exception { remoteOptions.remoteLocalFallback = true; RemoteSpawnRunner runner = newSpawnRunner(); when(cache.downloadActionResult( any(RemoteActionExecutionContext.class), any(ActionKey.class), /* inlineOutErr= */ eq(false))) .thenReturn(null); when(executor.executeRemotely( any(RemoteActionExecutionContext.class), any(ExecuteRequest.class), any(OperationObserver.class))) .thenThrow(new IOException()); Spawn spawn = newSimpleSpawn(); SpawnExecutionContext policy = getSpawnContext(spawn); IOException err = new IOException("local execution error"); when(localRunner.exec(eq(spawn), eq(policy))).thenThrow(err); IOException e = assertThrows(IOException.class, () -> runner.exec(spawn, policy)); assertThat(e).isSameInstanceAs(err); verify(localRunner).exec(eq(spawn), eq(policy)); } @Test public void testHumanReadableServerLogsSavedForFailingAction() throws Exception { RemoteSpawnRunner runner = newSpawnRunner(); RemoteExecutionService service = runner.getRemoteExecutionService(); Digest logDigest = digestUtil.computeAsUtf8("bla"); Path logPath = logDir.getRelative(SIMPLE_ACTION_ID).getRelative("logname"); ExecuteResponse resp = ExecuteResponse.newBuilder() .putServerLogs( "logname", LogFile.newBuilder().setHumanReadable(true).setDigest(logDigest).build()) .setResult(ActionResult.newBuilder().setExitCode(31).build()) .build(); when(executor.executeRemotely( any(RemoteActionExecutionContext.class), any(ExecuteRequest.class), any(OperationObserver.class))) .thenReturn(resp); SettableFuture completed = SettableFuture.create(); completed.set(null); when(cache.downloadFile(any(RemoteActionExecutionContext.class), eq(logPath), eq(logDigest))) .thenReturn(completed); Spawn spawn = newSimpleSpawn(); SpawnExecutionContext policy = getSpawnContext(spawn); SpawnResult res = runner.exec(spawn, policy); assertThat(res.status()).isEqualTo(Status.NON_ZERO_EXIT); verify(executor) .executeRemotely( any(RemoteActionExecutionContext.class), any(ExecuteRequest.class), any(OperationObserver.class)); verify(service).maybeDownloadServerLogs(any(), eq(resp), eq(logDir)); verify(cache).downloadFile(any(RemoteActionExecutionContext.class), eq(logPath), eq(logDigest)); } @Test public void testHumanReadableServerLogsSavedForFailingActionWithSiblingRepositoryLayout() throws Exception { RemoteSpawnRunner runner = newSpawnRunner(new SiblingRepositoryLayoutResolver(execRoot)); RemoteExecutionService service = runner.getRemoteExecutionService(); Digest logDigest = digestUtil.computeAsUtf8("bla"); Path logPath = logDir .getRelative("e0a5a3561464123504c1240b3587779cdfd6adee20f72aa136e388ecfd570c12") .getRelative("logname"); ExecuteResponse resp = ExecuteResponse.newBuilder() .putServerLogs( "logname", LogFile.newBuilder().setHumanReadable(true).setDigest(logDigest).build()) .setResult(ActionResult.newBuilder().setExitCode(31).build()) .build(); when(executor.executeRemotely( any(RemoteActionExecutionContext.class), any(ExecuteRequest.class), any(OperationObserver.class))) .thenReturn(resp); SettableFuture completed = SettableFuture.create(); completed.set(null); when(cache.downloadFile(any(RemoteActionExecutionContext.class), eq(logPath), eq(logDigest))) .thenReturn(completed); Spawn spawn = newSimpleSpawn(); SpawnExecutionContext policy = getSpawnContext(spawn); SpawnResult res = runner.exec(spawn, policy); assertThat(res.status()).isEqualTo(Status.NON_ZERO_EXIT); verify(executor) .executeRemotely( any(RemoteActionExecutionContext.class), any(ExecuteRequest.class), any(OperationObserver.class)); verify(service).maybeDownloadServerLogs(any(), eq(resp), eq(logDir)); verify(cache).downloadFile(any(RemoteActionExecutionContext.class), eq(logPath), eq(logDigest)); } @Test public void testHumanReadableServerLogsSavedForFailingActionWithStatus() throws Exception { RemoteSpawnRunner runner = newSpawnRunner(); RemoteExecutionService service = runner.getRemoteExecutionService(); Digest logDigest = digestUtil.computeAsUtf8("bla"); Path logPath = logDir.getRelative(SIMPLE_ACTION_ID).getRelative("logname"); com.google.rpc.Status timeoutStatus = com.google.rpc.Status.newBuilder().setCode(Code.DEADLINE_EXCEEDED.getNumber()).build(); ExecuteResponse resp = ExecuteResponse.newBuilder() .putServerLogs( "logname", LogFile.newBuilder().setHumanReadable(true).setDigest(logDigest).build()) .setStatus(timeoutStatus) .build(); when(executor.executeRemotely( any(RemoteActionExecutionContext.class), any(ExecuteRequest.class), any(OperationObserver.class))) .thenThrow(new IOException(new ExecutionStatusException(resp.getStatus(), resp))); SettableFuture completed = SettableFuture.create(); completed.set(null); when(cache.downloadFile(any(RemoteActionExecutionContext.class), eq(logPath), eq(logDigest))) .thenReturn(completed); Spawn spawn = newSimpleSpawn(); SpawnExecutionContext policy = getSpawnContext(spawn); SpawnResult res = runner.exec(spawn, policy); assertThat(res.status()).isEqualTo(Status.TIMEOUT); verify(executor) .executeRemotely( any(RemoteActionExecutionContext.class), any(ExecuteRequest.class), any(OperationObserver.class)); verify(service).maybeDownloadServerLogs(any(), eq(resp), eq(logDir)); verify(cache).downloadFile(any(RemoteActionExecutionContext.class), eq(logPath), eq(logDigest)); } @Test public void testNonHumanReadableServerLogsNotSaved() throws Exception { // arrange RemoteSpawnRunner runner = newSpawnRunner(); RemoteExecutionService service = runner.getRemoteExecutionService(); Digest logDigest = digestUtil.computeAsUtf8("bla"); ActionResult result = ActionResult.newBuilder().setExitCode(31).build(); ExecuteResponse resp = ExecuteResponse.newBuilder() .putServerLogs("logname", LogFile.newBuilder().setDigest(logDigest).build()) .setResult(result) .build(); when(executor.executeRemotely( any(RemoteActionExecutionContext.class), any(ExecuteRequest.class), any(OperationObserver.class))) .thenReturn(resp); Spawn spawn = newSimpleSpawn(); FakeSpawnExecutionContext policy = getSpawnContext(spawn); // act SpawnResult res = runner.exec(spawn, policy); // asset assertThat(res.status()).isEqualTo(Status.NON_ZERO_EXIT); verify(executor) .executeRemotely( any(RemoteActionExecutionContext.class), any(ExecuteRequest.class), any(OperationObserver.class)); verify(service).maybeDownloadServerLogs(any(), eq(resp), eq(logDir)); verify(cache, never()) .downloadFile(any(RemoteActionExecutionContext.class), any(Path.class), any(Digest.class)); } @Test public void testServerLogsNotSavedForSuccessfulAction() throws Exception { RemoteSpawnRunner runner = newSpawnRunner(); RemoteExecutionService service = runner.getRemoteExecutionService(); Digest logDigest = digestUtil.computeAsUtf8("bla"); ActionResult result = ActionResult.newBuilder().setExitCode(0).build(); ExecuteResponse resp = ExecuteResponse.newBuilder() .putServerLogs( "logname", LogFile.newBuilder().setHumanReadable(true).setDigest(logDigest).build()) .setResult(result) .build(); when(executor.executeRemotely( any(RemoteActionExecutionContext.class), any(ExecuteRequest.class), any(OperationObserver.class))) .thenReturn(resp); Spawn spawn = newSimpleSpawn(); FakeSpawnExecutionContext policy = getSpawnContext(spawn); SpawnResult res = runner.exec(spawn, policy); assertThat(res.status()).isEqualTo(Status.SUCCESS); verify(executor) .executeRemotely( any(RemoteActionExecutionContext.class), any(ExecuteRequest.class), any(OperationObserver.class)); verify(service).downloadOutputs(any(), eq(RemoteActionResult.createFromResponse(resp))); verify(service).maybeDownloadServerLogs(any(), eq(resp), eq(logDir)); verify(cache, never()) .downloadFile(any(RemoteActionExecutionContext.class), any(Path.class), any(Digest.class)); } @Test public void cacheDownloadFailureTriggersRemoteExecution() throws Exception { // If downloading a cached action fails, remote execution should be tried. // arrange RemoteSpawnRunner runner = newSpawnRunner(); RemoteExecutionService service = runner.getRemoteExecutionService(); CachedActionResult cachedResult = CachedActionResult.remote(ActionResult.newBuilder().setExitCode(0).build()); when(cache.downloadActionResult( any(RemoteActionExecutionContext.class), any(ActionKey.class), /* inlineOutErr= */ eq(false))) .thenReturn(cachedResult); Exception downloadFailure = new BulkTransferException(new CacheNotFoundException(Digest.getDefaultInstance())); doThrow(downloadFailure) .when(service) .downloadOutputs(any(), eq(RemoteActionResult.createFromCache(cachedResult))); ActionResult execResult = ActionResult.newBuilder().setExitCode(31).build(); ExecuteResponse succeeded = ExecuteResponse.newBuilder().setResult(execResult).build(); when(executor.executeRemotely( any(RemoteActionExecutionContext.class), any(ExecuteRequest.class), any(OperationObserver.class))) .thenReturn(succeeded); doReturn(null) .when(service) .downloadOutputs(any(), eq(RemoteActionResult.createFromResponse(succeeded))); Spawn spawn = newSimpleSpawn(); SpawnExecutionContext policy = getSpawnContext(spawn); // act SpawnResult res = runner.exec(spawn, policy); // assert assertThat(res.status()).isEqualTo(Status.NON_ZERO_EXIT); assertThat(res.exitCode()).isEqualTo(31); verify(executor) .executeRemotely( any(RemoteActionExecutionContext.class), any(ExecuteRequest.class), any(OperationObserver.class)); } @Test public void resultsDownloadFailureTriggersRemoteExecutionWithSkipCacheLookup() throws Exception { // If downloading an action result fails, remote execution should be retried // with skip cache lookup enabled // arrange RemoteSpawnRunner runner = newSpawnRunner(); RemoteExecutionService service = runner.getRemoteExecutionService(); when(cache.downloadActionResult( any(RemoteActionExecutionContext.class), any(ActionKey.class), /* inlineOutErr= */ eq(false))) .thenReturn(null); ActionResult cachedResult = ActionResult.newBuilder().setExitCode(0).build(); ActionResult execResult = ActionResult.newBuilder().setExitCode(31).build(); ExecuteResponse cachedResponse = ExecuteResponse.newBuilder().setResult(cachedResult).setCachedResult(true).build(); ExecuteResponse executedResponse = ExecuteResponse.newBuilder().setResult(execResult).build(); when(executor.executeRemotely( any(RemoteActionExecutionContext.class), any(ExecuteRequest.class), any(OperationObserver.class))) .thenReturn(cachedResponse) .thenReturn(executedResponse); Exception downloadFailure = new BulkTransferException(new CacheNotFoundException(Digest.getDefaultInstance())); doThrow(downloadFailure) .when(service) .downloadOutputs(any(), eq(RemoteActionResult.createFromResponse(cachedResponse))); doReturn(null) .when(service) .downloadOutputs(any(), eq(RemoteActionResult.createFromResponse(executedResponse))); Spawn spawn = newSimpleSpawn(); SpawnExecutionContext policy = getSpawnContext(spawn); // act SpawnResult res = runner.exec(spawn, policy); // assert assertThat(res.status()).isEqualTo(Status.NON_ZERO_EXIT); assertThat(res.exitCode()).isEqualTo(31); ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(ExecuteRequest.class); verify(executor, times(2)) .executeRemotely( any(RemoteActionExecutionContext.class), requestCaptor.capture(), any(OperationObserver.class)); List requests = requestCaptor.getAllValues(); // first request should have been executed without skip cache lookup assertThat(requests.get(0).getSkipCacheLookup()).isFalse(); // second should have been executed with skip cache lookup assertThat(requests.get(1).getSkipCacheLookup()).isTrue(); } @Test public void testRemoteExecutionTimeout() throws Exception { // If remote execution times out the SpawnResult status should be TIMEOUT. remoteOptions.remoteLocalFallback = false; RemoteSpawnRunner runner = newSpawnRunner(); RemoteExecutionService service = runner.getRemoteExecutionService(); ActionResult cachedResult = ActionResult.newBuilder().setExitCode(0).build(); when(cache.downloadActionResult( any(RemoteActionExecutionContext.class), any(ActionKey.class), /* inlineOutErr= */ eq(false))) .thenReturn(null); ExecuteResponse resp = ExecuteResponse.newBuilder() .setResult(cachedResult) .setStatus( com.google.rpc.Status.newBuilder() .setCode(Code.DEADLINE_EXCEEDED.getNumber()) .build()) .build(); when(executor.executeRemotely( any(RemoteActionExecutionContext.class), any(ExecuteRequest.class), any(OperationObserver.class))) .thenThrow(new IOException(new ExecutionStatusException(resp.getStatus(), resp))); Spawn spawn = newSimpleSpawn(); SpawnExecutionContext policy = getSpawnContext(spawn); SpawnResult res = runner.exec(spawn, policy); assertThat(res.status()).isEqualTo(Status.TIMEOUT); verify(executor) .executeRemotely( any(RemoteActionExecutionContext.class), any(ExecuteRequest.class), any(OperationObserver.class)); verify(service).downloadOutputs(any(), eq(RemoteActionResult.createFromResponse(resp))); } @Test public void testRemoteExecutionTimeoutTimings() throws Exception { // If remote execution times out the SpawnResult should still have the start and wall times // reported correctly. remoteOptions.remoteLocalFallback = false; RemoteSpawnRunner runner = newSpawnRunner(); RemoteExecutionService service = runner.getRemoteExecutionService(); com.google.protobuf.Duration oneSecond = Durations.fromMillis(1000); Timestamp executionStart = Timestamp.getDefaultInstance(); Timestamp executionCompleted = Timestamps.add(executionStart, oneSecond); ExecutedActionMetadata executedMetadata = ExecutedActionMetadata.newBuilder() .setExecutionStartTimestamp(executionStart) .setExecutionCompletedTimestamp(executionCompleted) .build(); ActionResult cachedResult = ActionResult.newBuilder().setExitCode(0).setExecutionMetadata(executedMetadata).build(); when(cache.downloadActionResult( any(RemoteActionExecutionContext.class), any(ActionKey.class), /* inlineOutErr= */ eq(false))) .thenReturn(null); ExecuteResponse resp = ExecuteResponse.newBuilder() .setResult(cachedResult) .setStatus( com.google.rpc.Status.newBuilder() .setCode(Code.DEADLINE_EXCEEDED.getNumber()) .build()) .build(); when(executor.executeRemotely( any(RemoteActionExecutionContext.class), any(ExecuteRequest.class), any(OperationObserver.class))) .thenThrow(new IOException(new ExecutionStatusException(resp.getStatus(), resp))); Spawn spawn = newSimpleSpawn(); SpawnExecutionContext policy = getSpawnContext(spawn); SpawnResult res = runner.exec(spawn, policy); assertThat(res.status()).isEqualTo(Status.TIMEOUT); assertThat(res.getWallTimeInMs()).isEqualTo(1000); assertThat(res.getStartTime()) .isEqualTo(Instant.ofEpochSecond(executionStart.getSeconds(), executionStart.getNanos())); verify(executor) .executeRemotely( any(RemoteActionExecutionContext.class), any(ExecuteRequest.class), any(OperationObserver.class)); verify(service).downloadOutputs(any(), eq(RemoteActionResult.createFromResponse(resp))); } @Test public void testRemoteExecutionTimeoutDoesNotTriggerFallback() throws Exception { // If remote execution times out the SpawnResult status should be TIMEOUT, regardess of local // fallback option. remoteOptions.remoteLocalFallback = true; RemoteSpawnRunner runner = newSpawnRunner(); RemoteExecutionService service = runner.getRemoteExecutionService(); ActionResult cachedResult = ActionResult.newBuilder().setExitCode(0).build(); when(cache.downloadActionResult( any(RemoteActionExecutionContext.class), any(ActionKey.class), /* inlineOutErr= */ eq(false))) .thenReturn(null); ExecuteResponse resp = ExecuteResponse.newBuilder() .setResult(cachedResult) .setStatus( com.google.rpc.Status.newBuilder() .setCode(Code.DEADLINE_EXCEEDED.getNumber()) .build()) .build(); when(executor.executeRemotely( any(RemoteActionExecutionContext.class), any(ExecuteRequest.class), any(OperationObserver.class))) .thenThrow(new IOException(new ExecutionStatusException(resp.getStatus(), resp))); Spawn spawn = newSimpleSpawn(); SpawnExecutionContext policy = getSpawnContext(spawn); SpawnResult res = runner.exec(spawn, policy); assertThat(res.status()).isEqualTo(Status.TIMEOUT); verify(executor) .executeRemotely( any(RemoteActionExecutionContext.class), any(ExecuteRequest.class), any(OperationObserver.class)); verify(service).downloadOutputs(any(), eq(RemoteActionResult.createFromResponse(resp))); verify(localRunner, never()).exec(eq(spawn), eq(policy)); } @Test public void testRemoteExecutionCommandFailureDoesNotTriggerFallback() throws Exception { remoteOptions.remoteLocalFallback = true; RemoteSpawnRunner runner = newSpawnRunner(); RemoteExecutionService service = runner.getRemoteExecutionService(); when(cache.downloadActionResult( any(RemoteActionExecutionContext.class), any(ActionKey.class), /* inlineOutErr= */ eq(false))) .thenReturn(null); ExecuteResponse failed = ExecuteResponse.newBuilder() .setResult(ActionResult.newBuilder().setExitCode(33).build()) .build(); when(executor.executeRemotely( any(RemoteActionExecutionContext.class), any(ExecuteRequest.class), any(OperationObserver.class))) .thenReturn(failed); Spawn spawn = newSimpleSpawn(); SpawnExecutionContext policy = getSpawnContext(spawn); SpawnResult res = runner.exec(spawn, policy); assertThat(res.status()).isEqualTo(Status.NON_ZERO_EXIT); assertThat(res.exitCode()).isEqualTo(33); verify(executor) .executeRemotely( any(RemoteActionExecutionContext.class), any(ExecuteRequest.class), any(OperationObserver.class)); verify(service).downloadOutputs(any(), eq(RemoteActionResult.createFromResponse(failed))); verify(localRunner, never()).exec(eq(spawn), eq(policy)); } @Test public void testExitCode_executorfailure() throws Exception { // If we get a failure due to the remote cache not working, the exit code should be // ExitCode.REMOTE_ERROR. remoteOptions.remoteLocalFallback = false; RemoteSpawnRunner runner = newSpawnRunner(); when(cache.downloadActionResult( any(RemoteActionExecutionContext.class), any(ActionKey.class), /* inlineOutErr= */ eq(false))) .thenReturn(null); when(executor.executeRemotely( any(RemoteActionExecutionContext.class), any(ExecuteRequest.class), any(OperationObserver.class))) .thenThrow(new IOException("reasons")); Spawn spawn = newSimpleSpawn(); SpawnExecutionContext policy = getSpawnContext(spawn); SpawnResult result = runner.exec(spawn, policy); assertThat(result.exitCode()).isEqualTo(ExitCode.REMOTE_ERROR.getNumericExitCode()); assertThat(result.getFailureMessage()).contains("reasons"); } @Test public void testExitCode_executionfailure() throws Exception { // If we get a failure due to the remote executor not working, the exit code should be // ExitCode.REMOTE_ERROR. remoteOptions.remoteLocalFallback = false; RemoteSpawnRunner runner = newSpawnRunner(); when(cache.downloadActionResult( any(RemoteActionExecutionContext.class), any(ActionKey.class), /* inlineOutErr= */ eq(false))) .thenThrow(new IOException("reasons")); Spawn spawn = newSimpleSpawn(); SpawnExecutionContext policy = getSpawnContext(spawn); SpawnResult result = runner.exec(spawn, policy); assertThat(result.exitCode()).isEqualTo(ExitCode.REMOTE_ERROR.getNumericExitCode()); assertThat(result.getFailureMessage()).contains("reasons"); } @Test public void testExitCode_remoteMessage() throws Exception { remoteOptions.remoteLocalFallback = false; RemoteSpawnRunner runner = newSpawnRunner(); ExecutionStatusException cause = new ExecutionStatusException( com.google.rpc.Status.getDefaultInstance(), ExecuteResponse.newBuilder().setMessage("beep and indeed boop").build()); when(cache.downloadActionResult( any(RemoteActionExecutionContext.class), any(ActionKey.class), /* inlineOutErr= */ eq(false))) .thenReturn(null); when(executor.executeRemotely( any(RemoteActionExecutionContext.class), any(ExecuteRequest.class), any(OperationObserver.class))) .thenThrow(new IOException("reasons", cause)); Spawn spawn = newSimpleSpawn(); SpawnExecutionContext policy = getSpawnContext(spawn); SpawnResult result = runner.exec(spawn, policy); assertThat(result.exitCode()).isEqualTo(ExitCode.REMOTE_ERROR.getNumericExitCode()); assertThat(result.getFailureMessage()).contains("beep and indeed boop"); } @Test public void testMaterializeParamFiles() throws Exception { testParamFilesAreMaterializedForFlag("--materialize_param_files"); } @Test public void testMaterializeParamFilesIsImpliedBySubcommands() throws Exception { testParamFilesAreMaterializedForFlag("--subcommands"); } private void testParamFilesAreMaterializedForFlag(String flag) throws Exception { RemoteOptions remoteOptions = Options.getDefaults(RemoteOptions.class); ExecutionOptions executionOptions = Options.parse(ExecutionOptions.class, flag).getOptions(); executionOptions.materializeParamFiles = true; RemoteExecutionService remoteExecutionService = new RemoteExecutionService( directExecutor(), reporter, /* verboseFailures= */ true, execRoot, RemotePathResolver.createDefault(execRoot), "build-req-id", "command-id", digestUtil, remoteOptions, cache, executor, tempPathGenerator, /* captureCorruptedOutputsDir= */ null, remoteOutputChecker); RemoteSpawnRunner runner = new RemoteSpawnRunner( execRoot, remoteOptions, executionOptions, true, /* cmdlineReporter= */ null, retryService, logDir, remoteExecutionService); ExecuteResponse succeeded = ExecuteResponse.newBuilder() .setResult(ActionResult.newBuilder().setExitCode(0).build()) .build(); when(executor.executeRemotely( any(RemoteActionExecutionContext.class), any(ExecuteRequest.class), any(OperationObserver.class))) .thenReturn(succeeded); ImmutableList args = ImmutableList.of("--foo", "--bar"); ParamFileActionInput input = new ParamFileActionInput( PathFragment.create("out/param_file"), args, ParameterFileType.UNQUOTED, ISO_8859_1); Spawn spawn = new SimpleSpawn( new FakeOwner("foo", "bar", "//dummy:label"), /* arguments= */ ImmutableList.of(), /* environment= */ ImmutableMap.of(), /* executionInfo= */ ImmutableMap.of(), /* inputs= */ NestedSetBuilder.create(Order.STABLE_ORDER, input), /* outputs= */ ImmutableSet.of(), ResourceSet.ZERO); SpawnExecutionContext policy = getSpawnContext(spawn); SpawnResult res = runner.exec(spawn, policy); assertThat(res.status()).isEqualTo(Status.SUCCESS); Path paramFile = execRoot.getRelative("out/param_file"); assertThat(paramFile.exists()).isTrue(); try (InputStream inputStream = paramFile.getInputStream()) { assertThat( new String(ByteStreams.toByteArray(inputStream), StandardCharsets.UTF_8).split("\n")) .asList() .containsExactly("--foo", "--bar"); } } @Test public void testDownloadMinimalOnCacheHit() throws Exception { // arrange remoteOptions.remoteOutputsMode = RemoteOutputsMode.MINIMAL; ActionResult succeededAction = ActionResult.newBuilder().setExitCode(0).build(); RemoteActionResult actionResult = RemoteActionResult.createFromCache(CachedActionResult.remote(succeededAction)); RemoteSpawnRunner runner = newSpawnRunner(); RemoteExecutionService service = runner.getRemoteExecutionService(); doReturn(actionResult).when(service).lookupCache(any()); doReturn(null).when(service).downloadOutputs(any(), any()); Spawn spawn = newSimpleSpawn(); SpawnExecutionContext policy = getSpawnContext(spawn); // act SpawnResult result = runner.exec(spawn, policy); assertThat(result.exitCode()).isEqualTo(0); assertThat(result.status()).isEqualTo(Status.SUCCESS); // assert verify(service).downloadOutputs(any(), eq(actionResult)); } @Test public void testDownloadMinimalOnCacheMiss() throws Exception { // arrange remoteOptions.remoteOutputsMode = RemoteOutputsMode.MINIMAL; ActionResult succeededAction = ActionResult.newBuilder().setExitCode(0).build(); ExecuteResponse succeeded = ExecuteResponse.newBuilder().setResult(succeededAction).build(); when(executor.executeRemotely( any(RemoteActionExecutionContext.class), any(ExecuteRequest.class), any(OperationObserver.class))) .thenReturn(succeeded); RemoteSpawnRunner runner = newSpawnRunner(); RemoteExecutionService service = runner.getRemoteExecutionService(); doReturn(null).when(service).downloadOutputs(any(), any()); Spawn spawn = newSimpleSpawn(); FakeSpawnExecutionContext policy = getSpawnContext(spawn); // act SpawnResult result = runner.exec(spawn, policy); assertThat(result.exitCode()).isEqualTo(0); assertThat(result.status()).isEqualTo(Status.SUCCESS); // assert verify(executor) .executeRemotely( any(RemoteActionExecutionContext.class), any(), any(OperationObserver.class)); verify(service).downloadOutputs(any(), eq(RemoteActionResult.createFromResponse(succeeded))); } @Test public void testDownloadMinimalIoError() throws Exception { // arrange remoteOptions.remoteOutputsMode = RemoteOutputsMode.MINIMAL; ActionResult succeededAction = ActionResult.newBuilder().setExitCode(0).build(); RemoteActionResult cachedActionResult = RemoteActionResult.createFromCache(CachedActionResult.remote(succeededAction)); IOException downloadFailure = new IOException("downloadMinimal failed"); RemoteSpawnRunner runner = newSpawnRunner(); RemoteExecutionService service = runner.getRemoteExecutionService(); doReturn(RemoteActionResult.createFromCache(CachedActionResult.remote(succeededAction))) .when(service) .lookupCache(any()); doThrow(downloadFailure).when(service).downloadOutputs(any(), eq(cachedActionResult)); Spawn spawn = newSimpleSpawn(); FakeSpawnExecutionContext policy = getSpawnContext(spawn); // act SpawnResult result = runner.exec(spawn, policy); assertThat(result.getFailureMessage()).isEqualTo(downloadFailure.getMessage()); // assert verify(service).downloadOutputs(any(), eq(cachedActionResult)); } @Test public void testDigest() throws Exception { RemoteSpawnRunner runner = newSpawnRunner(); RemoteExecutionService service = runner.getRemoteExecutionService(); ExecuteResponse resp = ExecuteResponse.newBuilder() .setResult(ActionResult.newBuilder().setExitCode(0).build()) .build(); when(executor.executeRemotely( any(RemoteActionExecutionContext.class), any(ExecuteRequest.class), any(OperationObserver.class))) .thenReturn(resp); Spawn spawn = newSimpleSpawn(); FakeSpawnExecutionContext policy = getSpawnContext(spawn); SpawnResult res = runner.exec(spawn, policy); assertThat(res.status()).isEqualTo(Status.SUCCESS); ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(RemoteAction.class); verify(service) .executeRemotely(requestCaptor.capture(), anyBoolean(), any(OperationObserver.class)); assertThat(res.getDigest()) .isEqualTo( SpawnResult.Digest.of( requestCaptor.getValue().getActionKey().getDigest().getHash(), requestCaptor.getValue().getActionKey().getDigest().getSizeBytes())); } @Test public void accountingDisabledWithoutWorker() { SpawnMetrics.Builder spawnMetrics = Mockito.mock(SpawnMetrics.Builder.class); RemoteSpawnRunner.spawnMetricsAccounting( spawnMetrics, ExecutedActionMetadata.getDefaultInstance()); verifyNoMoreInteractions(spawnMetrics); } @Test public void accountingAddsDurationsForStages() { SpawnMetrics.Builder builder = SpawnMetrics.Builder.forRemoteExec() .setQueueTimeInMs(1 * 1000) .setSetupTimeInMs(2 * 1000) .setExecutionWallTimeInMs(2 * 1000) .setProcessOutputsTimeInMs(2 * 1000); Timestamp queued = Timestamp.getDefaultInstance(); com.google.protobuf.Duration oneSecond = Durations.fromMillis(1000); Timestamp workerStart = Timestamps.add(queued, oneSecond); Timestamp executionStart = Timestamps.add(workerStart, oneSecond); Timestamp executionCompleted = Timestamps.add(executionStart, oneSecond); Timestamp outputUploadStart = Timestamps.add(executionCompleted, oneSecond); Timestamp outputUploadComplete = Timestamps.add(outputUploadStart, oneSecond); ExecutedActionMetadata executedMetadata = ExecutedActionMetadata.newBuilder() .setWorker("test worker") .setQueuedTimestamp(queued) .setWorkerStartTimestamp(workerStart) .setExecutionStartTimestamp(executionStart) .setExecutionCompletedTimestamp(executionCompleted) .setOutputUploadStartTimestamp(outputUploadStart) .setOutputUploadCompletedTimestamp(outputUploadComplete) .build(); RemoteSpawnRunner.spawnMetricsAccounting(builder, executedMetadata); SpawnMetrics spawnMetrics = builder.build(); // remote queue time is accumulated assertThat(spawnMetrics.queueTimeInMs()).isEqualTo(2 * 1000L); // setup time is substituted assertThat(spawnMetrics.setupTimeInMs()).isEqualTo(1 * 1000L); // execution time is unspecified, assume substituted assertThat(spawnMetrics.executionWallTimeInMs()).isEqualTo(1 * 1000L); // ProcessOutputs time is unspecified, assume substituted assertThat(spawnMetrics.processOutputsTimeInMs()).isEqualTo(1 * 1000L); } @Test public void shouldReportCheckingCacheBeforeScheduling() throws Exception { // Arrange RemoteSpawnRunner runner = newSpawnRunner(); RemoteExecutionService service = runner.getRemoteExecutionService(); ExecuteResponse succeeded = ExecuteResponse.newBuilder() .setResult(ActionResult.newBuilder().setExitCode(0).build()) .build(); Spawn spawn = newSimpleSpawn(); SpawnExecutionContext policy = mock(SpawnExecutionContext.class); when(policy.getTimeout()).thenReturn(Duration.ZERO); when(executor.executeRemotely( any(RemoteActionExecutionContext.class), any(ExecuteRequest.class), any(OperationObserver.class))) .thenAnswer( invocationOnMock -> { OperationObserver receiver = invocationOnMock.getArgument(2); receiver.onNext(Operation.getDefaultInstance()); return succeeded; }); doReturn(null).when(service).downloadOutputs(any(), any()); // Act SpawnResult res = runner.exec(spawn, policy); assertThat(res.status()).isEqualTo(Status.SUCCESS); // Assert verify(executor) .executeRemotely( any(RemoteActionExecutionContext.class), any(ExecuteRequest.class), any(OperationObserver.class)); InOrder reportOrder = inOrder(policy); reportOrder.verify(policy, times(1)).report(SpawnCheckingCacheEvent.create("remote")); reportOrder.verify(policy, times(1)).report(SpawnSchedulingEvent.create("remote")); reportOrder.verify(policy, times(1)).report(SpawnExecutingEvent.create("remote")); } @Test public void shouldReportExecutingStatusWithoutMetadata() throws Exception { // arrange RemoteSpawnRunner runner = newSpawnRunner(); RemoteExecutionService service = runner.getRemoteExecutionService(); ExecuteResponse succeeded = ExecuteResponse.newBuilder() .setResult(ActionResult.newBuilder().setExitCode(0).build()) .build(); Spawn spawn = newSimpleSpawn(); SpawnExecutionContext policy = mock(SpawnExecutionContext.class); when(policy.getTimeout()).thenReturn(Duration.ZERO); when(executor.executeRemotely( any(RemoteActionExecutionContext.class), any(ExecuteRequest.class), any(OperationObserver.class))) .thenAnswer( invocationOnMock -> { OperationObserver receiver = invocationOnMock.getArgument(2); verify(policy, never()).report(SpawnExecutingEvent.create("remote")); receiver.onNext(Operation.getDefaultInstance()); return succeeded; }); doReturn(null).when(service).downloadOutputs(any(), any()); // act SpawnResult res = runner.exec(spawn, policy); assertThat(res.status()).isEqualTo(Status.SUCCESS); // assert verify(executor) .executeRemotely( any(RemoteActionExecutionContext.class), any(ExecuteRequest.class), any(OperationObserver.class)); InOrder reportOrder = inOrder(policy); reportOrder.verify(policy, times(1)).report(SpawnSchedulingEvent.create("remote")); reportOrder.verify(policy, times(1)).report(SpawnExecutingEvent.create("remote")); } @Test public void shouldReportExecutingStatusAfterGotExecutingStageFromMetadata() throws Exception { // arrange RemoteSpawnRunner runner = newSpawnRunner(); RemoteExecutionService service = runner.getRemoteExecutionService(); ExecuteResponse succeeded = ExecuteResponse.newBuilder() .setResult(ActionResult.newBuilder().setExitCode(0).build()) .build(); Spawn spawn = newSimpleSpawn(); SpawnExecutionContext policy = mock(SpawnExecutionContext.class); when(policy.getTimeout()).thenReturn(Duration.ZERO); when(executor.executeRemotely( any(RemoteActionExecutionContext.class), any(ExecuteRequest.class), any(OperationObserver.class))) .thenAnswer( invocationOnMock -> { OperationObserver receiver = invocationOnMock.getArgument(2); Operation queued = Operation.newBuilder() .setMetadata( Any.pack( ExecuteOperationMetadata.newBuilder().setStage(Value.QUEUED).build())) .build(); receiver.onNext(queued); verify(policy, never()).report(SpawnExecutingEvent.create("remote")); Operation executing = Operation.newBuilder() .setMetadata( Any.pack( ExecuteOperationMetadata.newBuilder() .setStage(Value.EXECUTING) .build())) .build(); receiver.onNext(executing); return succeeded; }); doReturn(null).when(service).downloadOutputs(any(), any()); // act SpawnResult res = runner.exec(spawn, policy); assertThat(res.status()).isEqualTo(Status.SUCCESS); // assert verify(executor) .executeRemotely( any(RemoteActionExecutionContext.class), any(ExecuteRequest.class), any(OperationObserver.class)); InOrder reportOrder = inOrder(policy); reportOrder.verify(policy, times(1)).report(SpawnSchedulingEvent.create("remote")); reportOrder.verify(policy, times(1)).report(SpawnExecutingEvent.create("remote")); } @Test public void shouldIgnoreInvalidMetadata() throws Exception { // arrange RemoteSpawnRunner runner = newSpawnRunner(); RemoteExecutionService service = runner.getRemoteExecutionService(); ExecuteResponse succeeded = ExecuteResponse.newBuilder() .setResult(ActionResult.newBuilder().setExitCode(0).build()) .build(); Spawn spawn = newSimpleSpawn(); SpawnExecutionContext policy = mock(SpawnExecutionContext.class); when(policy.getTimeout()).thenReturn(Duration.ZERO); when(executor.executeRemotely( any(RemoteActionExecutionContext.class), any(ExecuteRequest.class), any(OperationObserver.class))) .thenAnswer( invocationOnMock -> { OperationObserver receiver = invocationOnMock.getArgument(2); Operation operation = Operation.newBuilder() .setMetadata( // Anything that is not ExecutionOperationMetadata Any.pack(Operation.getDefaultInstance())) .build(); receiver.onNext(operation); return succeeded; }); doReturn(null).when(service).downloadOutputs(any(), any()); // act SpawnResult res = runner.exec(spawn, policy); assertThat(res.status()).isEqualTo(Status.SUCCESS); // assert verify(executor) .executeRemotely( any(RemoteActionExecutionContext.class), any(ExecuteRequest.class), any(OperationObserver.class)); InOrder reportOrder = inOrder(policy); reportOrder.verify(policy, times(1)).report(SpawnSchedulingEvent.create("remote")); reportOrder.verify(policy, times(1)).report(SpawnExecutingEvent.create("remote")); } @Test public void shouldReportExecutingStatusIfNoExecutingStatusFromMetadata() throws Exception { // arrange RemoteSpawnRunner runner = newSpawnRunner(); RemoteExecutionService service = runner.getRemoteExecutionService(); ExecuteResponse succeeded = ExecuteResponse.newBuilder() .setResult(ActionResult.newBuilder().setExitCode(0).build()) .build(); Spawn spawn = newSimpleSpawn(); SpawnExecutionContext policy = mock(SpawnExecutionContext.class); when(policy.getTimeout()).thenReturn(Duration.ZERO); when(executor.executeRemotely( any(RemoteActionExecutionContext.class), any(ExecuteRequest.class), any(OperationObserver.class))) .thenAnswer( invocationOnMock -> { OperationObserver receiver = invocationOnMock.getArgument(2); Operation completed = Operation.newBuilder() .setMetadata( Any.pack( ExecuteOperationMetadata.newBuilder() .setStage(Value.COMPLETED) .build())) .build(); receiver.onNext(completed); return succeeded; }); doReturn(null).when(service).downloadOutputs(any(), any()); // act SpawnResult res = runner.exec(spawn, policy); assertThat(res.status()).isEqualTo(Status.SUCCESS); // assert verify(executor) .executeRemotely( any(RemoteActionExecutionContext.class), any(ExecuteRequest.class), any(OperationObserver.class)); InOrder reportOrder = inOrder(policy); reportOrder.verify(policy, times(1)).report(SpawnSchedulingEvent.create("remote")); reportOrder.verify(policy, times(1)).report(SpawnExecutingEvent.create("remote")); } @Test public void shouldReportExecutingStatusEvenNoOperationFromServer() throws Exception { // arrange RemoteSpawnRunner runner = newSpawnRunner(); RemoteExecutionService service = runner.getRemoteExecutionService(); ExecuteResponse succeeded = ExecuteResponse.newBuilder() .setResult(ActionResult.newBuilder().setExitCode(0).build()) .build(); Spawn spawn = newSimpleSpawn(); SpawnExecutionContext policy = mock(SpawnExecutionContext.class); when(policy.getTimeout()).thenReturn(Duration.ZERO); when(executor.executeRemotely( any(RemoteActionExecutionContext.class), any(ExecuteRequest.class), any(OperationObserver.class))) .thenReturn(succeeded); doReturn(null).when(service).downloadOutputs(any(), any()); // act SpawnResult res = runner.exec(spawn, policy); assertThat(res.status()).isEqualTo(Status.SUCCESS); // assert verify(executor) .executeRemotely( any(RemoteActionExecutionContext.class), any(ExecuteRequest.class), any(OperationObserver.class)); InOrder reportOrder = inOrder(policy); reportOrder.verify(policy, times(1)).report(SpawnSchedulingEvent.create("remote")); reportOrder.verify(policy, times(1)).report(SpawnExecutingEvent.create("remote")); } private static Spawn newSimpleSpawn(Artifact... outputs) { return simpleSpawnWithExecutionInfo(ImmutableMap.of(), outputs); } private static SimpleSpawn simpleSpawnWithExecutionInfo( ImmutableMap executionInfo, Artifact... outputs) { return new SimpleSpawn( new FakeOwner("foo", "bar", "//dummy:label"), /* arguments= */ ImmutableList.of(), /* environment= */ ImmutableMap.of(), /* executionInfo= */ executionInfo, /* inputs= */ NestedSetBuilder.emptySet(Order.STABLE_ORDER), /* outputs= */ ImmutableSet.copyOf(outputs), ResourceSet.ZERO); } private RemoteSpawnRunner newSpawnRunner() { return newSpawnRunner(executor, RemotePathResolver.createDefault(execRoot)); } private RemoteSpawnRunner newSpawnRunner(RemotePathResolver remotePathResolver) { return newSpawnRunner(executor, remotePathResolver); } private RemoteSpawnRunner newSpawnRunner( @Nullable RemoteExecutionClient executor, RemotePathResolver remotePathResolver) { RemoteExecutionService service = spy( new RemoteExecutionService( directExecutor(), reporter, /* verboseFailures= */ true, execRoot, remotePathResolver, "build-req-id", "command-id", digestUtil, remoteOptions, cache, executor, tempPathGenerator, /* captureCorruptedOutputsDir= */ null, remoteOutputChecker)); return new RemoteSpawnRunner( execRoot, remoteOptions, Options.getDefaults(ExecutionOptions.class), false, reporter, retryService, logDir, service); } }