Skip to content

Commit a21c882

Browse files
labkey-teamcitylabkey-teamcity
committed
Merge 24.11 to develop
2 parents 359bdb9 + 3e81970 commit a21c882

5 files changed

Lines changed: 170 additions & 26 deletions

File tree

‎nextflow/src/org/labkey/nextflow/NextFlowController.java‎

Lines changed: 22 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
import lombok.Getter;
44
import lombok.Setter;
55
import org.apache.commons.lang3.StringUtils;
6-
import org.apache.logging.log4j.Logger;
76
import org.labkey.api.action.ApiResponse;
87
import org.labkey.api.action.ApiSimpleResponse;
98
import org.labkey.api.action.FormViewAction;
@@ -14,6 +13,7 @@
1413
import org.labkey.api.data.PropertyStore;
1514
import org.labkey.api.pipeline.PipeRoot;
1615
import org.labkey.api.pipeline.PipelineJob;
16+
import org.labkey.api.pipeline.PipelineProvider;
1717
import org.labkey.api.pipeline.PipelineService;
1818
import org.labkey.api.pipeline.PipelineStatusUrls;
1919
import org.labkey.api.pipeline.browse.PipelinePathForm;
@@ -31,13 +31,13 @@
3131
import org.labkey.api.util.Path;
3232
import org.labkey.api.util.URLHelper;
3333
import org.labkey.api.util.element.Select;
34-
import org.labkey.api.util.logging.LogHelper;
3534
import org.labkey.api.view.HtmlView;
3635
import org.labkey.api.view.JspView;
3736
import org.labkey.api.view.NavTree;
3837
import org.labkey.api.view.UnauthorizedException;
3938
import org.labkey.api.view.ViewBackgroundInfo;
4039
import org.labkey.nextflow.pipeline.NextFlowPipelineJob;
40+
import org.labkey.nextflow.pipeline.NextFlowProtocol;
4141
import org.springframework.validation.BindException;
4242
import org.springframework.validation.Errors;
4343
import org.springframework.web.servlet.ModelAndView;
@@ -64,8 +64,6 @@ public class NextFlowController extends SpringActionController
6464
private static final DefaultActionResolver _actionResolver = new DefaultActionResolver(NextFlowController.class);
6565
public static final String NAME = "nextflow";
6666

67-
private static final Logger LOG = LogHelper.getLogger(NextFlowController.class, NAME);
68-
6967
public NextFlowController()
7068
{
7169
setActionResolver(_actionResolver);
@@ -262,24 +260,39 @@ public void validateCommand(AnalyzeForm o, Errors errors)
262260
@Override
263261
public ModelAndView getView(AnalyzeForm o, boolean b, BindException errors)
264262
{
263+
List<File> selectedFiles = o.getValidatedFiles(getContainer(), false);
264+
if (selectedFiles.isEmpty())
265+
{
266+
return new HtmlView(HtmlString.of("Couldn't find input file(s)"));
267+
}
268+
// NextFlow operates on the full directory so show the list to the user, regardless of what they selected
269+
// from the file listing
270+
File inputDir = selectedFiles.get(0).getParentFile();
271+
272+
File[] inputFiles = inputDir.listFiles(new PipelineProvider.FileTypesEntryFilter(NextFlowProtocol.INPUT_TYPES));
273+
if (inputFiles == null || inputFiles.length == 0)
274+
{
275+
return new HtmlView(HtmlString.of("Couldn't find input file(s)"));
276+
}
277+
265278
NextFlowConfiguration config = NextFlowManager.get().getConfiguration();
266279
if (config.getNextFlowConfigFilePath() != null)
267280
{
268281
File configDir = new File(config.getNextFlowConfigFilePath());
269282
if (configDir.isDirectory())
270283
{
271-
File[] files = configDir.listFiles();
272-
if (files != null && files.length > 0)
284+
File[] configFiles = configDir.listFiles();
285+
if (configFiles != null && configFiles.length > 0)
273286
{
274-
List<File> configFiles = Arrays.asList(files);
275287
return new HtmlView("NextFlow Runner", DIV(
276288
FORM(at(method, "POST"),
277289
INPUT(at(hidden, true, name, "launch", value, true)),
278290
Arrays.stream(o.getFile()).map(f -> INPUT(at(hidden, true, name, "file", value, f))).toList(),
279291
"Files: ",
280-
UL(Arrays.stream(o.getFile()).map(DOM::LI)),
292+
UL(Arrays.stream(inputFiles).map(File::getName).map(DOM::LI)),
281293
"Config: ",
282-
new Select.SelectBuilder().name("configFile").addOptions(configFiles.stream().filter(f -> f.isFile() && f.getName().toLowerCase().endsWith(".config")).map(File::getName).sorted(String.CASE_INSENSITIVE_ORDER).toList()).build(),
294+
new Select.SelectBuilder().name("configFile").addOptions(Arrays.stream(configFiles).filter(f -> f.isFile() && f.getName().toLowerCase().endsWith(".config")).map(File::getName).sorted(String.CASE_INSENSITIVE_ORDER).toList()).build(),
295+
DOM.BR(),
283296
new Button.ButtonBuilder("Start NextFlow").submit(true).build())));
284297
}
285298
}

‎nextflow/src/org/labkey/nextflow/pipeline/NextFlowPipelineJob.java‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -59,24 +59,26 @@ public NextFlowPipelineJob(ViewBackgroundInfo info, @NotNull PipeRoot root, Path
5959
super(new NextFlowProtocol(), NextFlowPipelineProvider.NAME, info, root, config.getFileName().toString(), config, inputFiles, false, false);
6060
this.config = config;
6161
setLogFile(log);
62-
LOG.info("NextFlow job queued: {}", getJsonJobInfo());
62+
LOG.info("NextFlow job queued: {}", getJsonJobInfo(null));
6363
}
6464

65-
protected JSONObject getJsonJobInfo()
65+
protected JSONObject getJsonJobInfo(Long invocationCount)
6666
{
6767
JSONObject result = new JSONObject();
6868
result.put("user", getUser().getEmail());
6969
result.put("container", getContainer().getPath());
7070
result.put("filePath", getLogFilePath().getParent().toString());
71-
result.put("runName", getNextFlowRunName());
71+
result.put("runName", getNextFlowRunName(invocationCount));
7272
result.put("configFile", getConfig().getFileName().toString());
7373
return result;
7474
}
7575

76-
protected String getNextFlowRunName()
76+
protected String getNextFlowRunName(Long invocationCount)
7777
{
7878
PipelineStatusFile file = PipelineService.get().getStatusFile(getJobGUID());
79-
return file == null ? "Unknown" : ("LabKeyJob" + file.getRowId());
79+
String result = file == null ? "Unknown" : ("LabKeyJob" + file.getRowId());
80+
result += invocationCount == null ? "" : ("_" + invocationCount);
81+
return result;
8082
}
8183

8284
@Override

‎nextflow/src/org/labkey/nextflow/pipeline/NextFlowRunTask.java‎

Lines changed: 19 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,9 @@
22

33
import org.apache.logging.log4j.Logger;
44
import org.jetbrains.annotations.NotNull;
5+
import org.labkey.api.data.ContainerManager;
6+
import org.labkey.api.data.DbSequence;
7+
import org.labkey.api.data.DbSequenceManager;
58
import org.labkey.api.exp.XarFormatException;
69
import org.labkey.api.pipeline.AbstractTaskFactory;
710
import org.labkey.api.pipeline.AbstractTaskFactorySettings;
@@ -37,6 +40,8 @@ public class NextFlowRunTask extends WorkDirectoryTask<NextFlowRunTask.Factory>
3740

3841
public static final String ACTION_NAME = "NextFlow";
3942

43+
private static final DbSequence INVOCATION_SEQUENCE = DbSequenceManager.get(ContainerManager.getRoot(), NextFlowRunTask.class.getName());
44+
4045
public NextFlowRunTask(Factory factory, PipelineJob job)
4146
{
4247
super(factory, job);
@@ -46,7 +51,12 @@ public NextFlowRunTask(Factory factory, PipelineJob job)
4651
public @NotNull RecordedActionSet run() throws PipelineJobException
4752
{
4853
Logger log = getJob().getLogger();
49-
NextFlowPipelineJob.LOG.info("Starting to execute NextFlow: {}", getJob().getJsonJobInfo());
54+
55+
// NextFlow requires a unique job name for every execution. Increment a counter to append as a suffix to
56+
// ensure uniqueness
57+
long invocationCount = INVOCATION_SEQUENCE.next();
58+
INVOCATION_SEQUENCE.sync();
59+
NextFlowPipelineJob.LOG.info("Starting to execute NextFlow: {}", getJob().getJsonJobInfo(invocationCount));
5060

5161
SecurityManager.TransformSession session = null;
5262
boolean success = false;
@@ -73,10 +83,10 @@ public NextFlowRunTask(Factory factory, PipelineJob job)
7383
File dir = getJob().getLogFile().getParentFile();
7484
getJob().runSubProcess(secretsPB, dir);
7585

76-
ProcessBuilder executionPB = new ProcessBuilder(getArgs());
86+
ProcessBuilder executionPB = new ProcessBuilder(getArgs(invocationCount));
7787
getJob().runSubProcess(executionPB, dir);
7888
log.info("Job Finished");
79-
NextFlowPipelineJob.LOG.info("Finished executing NextFlow: {}", getJob().getJsonJobInfo());
89+
NextFlowPipelineJob.LOG.info("Finished executing NextFlow: {}", getJob().getJsonJobInfo(invocationCount));
8090

8191
RecordedAction action = new RecordedAction(ACTION_NAME);
8292
for (Path inputFile : getJob().getInputFilePaths())
@@ -100,14 +110,16 @@ public NextFlowRunTask(Factory factory, PipelineJob job)
100110
}
101111
if (!success)
102112
{
103-
NextFlowPipelineJob.LOG.info("Failed executing NextFlow: {}", getJob().getJsonJobInfo());
113+
NextFlowPipelineJob.LOG.info("Failed executing NextFlow: {}", getJob().getJsonJobInfo(invocationCount));
104114
}
105115
}
106116
}
107117

108118
private void addOutputs(RecordedAction action, Path path, Logger log) throws IOException
109119
{
110-
if (Files.isRegularFile(path))
120+
// Skip results.sky.zip files - it's the template document. We want the file output doc that includes
121+
// the replicate analysis
122+
if (Files.isRegularFile(path) && !path.endsWith("results.sky.zip"))
111123
{
112124
action.addOutput(path.toFile(), "Output", false);
113125
if (path.toString().toLowerCase().endsWith(".sky.zip"))
@@ -164,7 +176,7 @@ private boolean hasAwsSection(Path configFile) throws PipelineJobException
164176
}
165177

166178

167-
private @NotNull List<String> getArgs() throws PipelineJobException
179+
private @NotNull List<String> getArgs(long invocationCount) throws PipelineJobException
168180
{
169181
NextFlowConfiguration config = NextFlowManager.get().getConfiguration();
170182
Path configFile = getJob().getConfig();
@@ -189,7 +201,7 @@ private boolean hasAwsSection(Path configFile) throws PipelineJobException
189201
args.add("-c");
190202
args.add(configFile.toAbsolutePath().toString());
191203
args.add("-name");
192-
args.add(getJob().getNextFlowRunName());
204+
args.add(getJob().getNextFlowRunName(invocationCount));
193205
return args;
194206
}
195207

‎panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PanoramaPublicBaseTest.java‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -306,7 +306,7 @@ private void submitForm()
306306
clickAndWait(Locator.linkWithText("Back to Experiment Details")); // Navigate to the experiment details page.
307307
}
308308

309-
void copyExperimentAndVerify(String projectName, String folderName, String experimentTitle, String destinationFolder, String shortAccessUrl)
309+
void copyExperimentAndVerify(String projectName, @Nullable String folderName, String experimentTitle, String destinationFolder, String shortAccessUrl)
310310
{
311311
copyExperimentAndVerify(projectName, folderName, null, experimentTitle, null, false, true, destinationFolder, shortAccessUrl);
312312
}
@@ -325,15 +325,15 @@ void makeCopy(String shortAccessUrl, String experimentTitle, String destinationF
325325
makeCopy(shortAccessUrl, experimentTitle, recopy, deleteOldCopy, destinationFolder, true);
326326
}
327327

328-
void copyExperimentAndVerify(String projectName, String folderName, @Nullable List<String> subfolders, String experimentTitle,
328+
void copyExperimentAndVerify(String projectName, @Nullable String folderName, @Nullable List<String> subfolders, String experimentTitle,
329329
@Nullable Integer version, boolean recopy, boolean deleteOldCopy, String destinationFolder,
330330
String shortAccessUrl)
331331
{
332332
copyExperimentAndVerify(projectName, folderName, subfolders, experimentTitle, version, recopy, deleteOldCopy,
333333
destinationFolder, shortAccessUrl, true);
334334
}
335335

336-
void copyExperimentAndVerify(String projectName, String folderName, @Nullable List<String> subfolders, String experimentTitle,
336+
void copyExperimentAndVerify(String projectName, @Nullable String folderName, @Nullable List<String> subfolders, String experimentTitle,
337337
@Nullable Integer version, boolean recopy, boolean deleteOldCopy, String destinationFolder,
338338
String shortAccessUrl, boolean symlinks)
339339
{
@@ -406,7 +406,7 @@ protected final void uncheck(String label)
406406
}
407407
}
408408

409-
private void verifyCopy(String shortAccessUrl, String experimentTitle, @Nullable Integer version, String projectName, String folderName, List<String> subfolders, boolean recopy)
409+
private void verifyCopy(String shortAccessUrl, String experimentTitle, @Nullable Integer version, String projectName, @Nullable String folderName, List<String> subfolders, boolean recopy)
410410
{
411411
// Verify the copy
412412
goToProjectHome(PANORAMA_PUBLIC);
@@ -454,7 +454,7 @@ private void verifyCopy(String shortAccessUrl, String experimentTitle, @Nullable
454454
text += "\nEmail: " + REVIEWER_PREFIX;
455455
}
456456
String messageText = new BodyWebPart(getDriver(), "View Message").getComponentElement().getText();
457-
var srcFolderTxt = "Source folder: " + "/" + projectName + "/" + folderName;
457+
var srcFolderTxt = "Source folder: " + "/" + projectName + (folderName == null ? "" : "/" + folderName);
458458
assertTextPresent(new TextSearcher(messageText), text, srcFolderTxt);
459459

460460
// Unescaped special Markdown characters in the message may cause the password to render incorrectly.

‎panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PanoramaPublicMoveSkyDocTest.java‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
package org.labkey.test.tests.panoramapublic;
2+
3+
import org.junit.Assert;
4+
import org.junit.Test;
5+
import org.junit.experimental.categories.Category;
6+
import org.labkey.test.BaseWebDriverTest;
7+
import org.labkey.test.Locator;
8+
import org.labkey.test.TestFileUtils;
9+
import org.labkey.test.categories.External;
10+
import org.labkey.test.categories.MacCossLabModules;
11+
import org.labkey.test.components.CustomizeView;
12+
import org.labkey.test.util.DataRegionTable;
13+
14+
import static org.junit.Assert.fail;
15+
16+
17+
@Category({External.class, MacCossLabModules.class})
18+
@BaseWebDriverTest.ClassTimeout(minutes = 5)
19+
public class PanoramaPublicMoveSkyDocTest extends PanoramaPublicBaseTest
20+
{
21+
private static final String SKY_FILE_1 = "MRMer.zip";
22+
private static final String SKY_FILE_2 = "MRMer_renamed_protein.zip";
23+
private static final String SKY_FILE_3 = "SmMolLibA.sky.zip";
24+
25+
@Test
26+
public void testExperimentCopy()
27+
{
28+
String projectName = getProjectName();
29+
String sourceFolder = "SourceFolder";
30+
String sourceSubfolder = "SourceSubFolder";
31+
String targetFolder = "TargetFolder";
32+
33+
log("Creating source folder " + sourceFolder);
34+
setupSourceFolder(projectName, sourceFolder, SUBMITTER);
35+
log("Creating subfolder, " + sourceSubfolder + " in folder " + sourceFolder);
36+
setupSubfolder(projectName, sourceFolder, sourceSubfolder, FolderType.Experiment, SUBMITTER);
37+
38+
goToProjectHome();
39+
log("Creating target folder " + targetFolder);
40+
setupSourceFolder(projectName, targetFolder, SUBMITTER);
41+
42+
impersonate(SUBMITTER);
43+
updateSubmitterAccountInfo("One");
44+
45+
goToProjectFolder(projectName, sourceFolder);
46+
log("Importing " + SKY_FILE_1 + " in folder " + sourceFolder);
47+
importData(SKY_FILE_1, 1);
48+
goToDashboard();
49+
log("Moving " + SKY_FILE_1 + " FROM " + sourceFolder + " TO " + targetFolder);
50+
moveDocument(SKY_FILE_1, targetFolder, 1);
51+
52+
var skyDocSourceFolder = sourceFolder + "/" + sourceSubfolder;
53+
goToProjectFolder(projectName, skyDocSourceFolder);
54+
log("Importing " + SKY_FILE_2 + " in folder " + skyDocSourceFolder);
55+
importData(SKY_FILE_2, 1);
56+
goToDashboard();
57+
log("Moving " + SKY_FILE_2 + " FROM " + skyDocSourceFolder + "TO " + targetFolder);
58+
moveDocument(SKY_FILE_2, targetFolder, 2);
59+
60+
goToProjectFolder(projectName, targetFolder);
61+
log("Importing " + SKY_FILE_3 + " in folder " + skyDocSourceFolder);
62+
importData(SKY_FILE_3, 3);
63+
64+
log("Creating and submitting an experiment");
65+
String experimentTitle = "Experiment to test moving Skyline documents from other folders";
66+
var expWebPart = createExperimentCompleteMetadata(experimentTitle);
67+
expWebPart.clickSubmit();
68+
String shortAccessLink = submitWithoutPXId();
69+
70+
// Copy the experiment to the Panorama Public project
71+
var panoramaCopyFolder = "Copy of " + targetFolder;
72+
log("Copying experiment to folder " + panoramaCopyFolder +" in the Panorama Public project");
73+
copyExperimentAndVerify(projectName, targetFolder, experimentTitle, panoramaCopyFolder, shortAccessLink);
74+
goToProjectFolder(PANORAMA_PUBLIC, panoramaCopyFolder);
75+
verifyRunFilePathRoot(SKY_FILE_1, PANORAMA_PUBLIC, panoramaCopyFolder);
76+
verifyRunFilePathRoot(SKY_FILE_2, PANORAMA_PUBLIC, panoramaCopyFolder);
77+
verifyRunFilePathRoot(SKY_FILE_3, PANORAMA_PUBLIC, panoramaCopyFolder);
78+
}
79+
80+
private void moveDocument(String skylineDocName, String targetFolder, int jobCount)
81+
{
82+
DataRegionTable table = new DataRegionTable.DataRegionFinder(getDriver()).withName("TargetedMSRuns").waitFor();
83+
table.checkCheckbox(0);
84+
table.clickHeaderButton("Move");
85+
waitAndClickAndWait(Locator.linkWithText(targetFolder));
86+
waitForPipelineJobsToComplete(jobCount, false);
87+
88+
goToDashboard();
89+
90+
// Verify that the document moved
91+
table = new DataRegionTable.DataRegionFinder(getDriver()).withName("TargetedMSRuns").waitFor();
92+
int rowIndex = table.getRowIndex("File", skylineDocName);
93+
if (rowIndex < 0)
94+
fail("Unable to find row for moved Skyline document: " + skylineDocName);
95+
96+
verifyRunFilePathRoot(skylineDocName, getProjectName(), targetFolder);
97+
}
98+
99+
private void verifyRunFilePathRoot(String skylineDocName, String projectName, String targetFolder)
100+
{
101+
// Verify that exp.run filePathRoot is set to the target folder
102+
portalHelper.navigateToQuery("exp", "Runs");
103+
DataRegionTable queryGrid = new DataRegionTable("query", this);
104+
CustomizeView view = queryGrid.openCustomizeGrid();
105+
view.showHiddenItems();
106+
view.addColumn("FilePathRoot");
107+
view.applyCustomView();
108+
int rowIndex = queryGrid.getRowIndex("Name", skylineDocName);
109+
if (rowIndex < 0)
110+
fail("Unable to find row for Skyline document in exp.Runs query grid: " + skylineDocName);
111+
112+
var expectedFileRoot = TestFileUtils.getDefaultFileRoot(projectName + "/" + targetFolder).getPath();
113+
var filePathRoot = queryGrid.getDataAsText(rowIndex, "FilePathRoot");
114+
115+
Assert.assertEquals("Unexpected FilePathRoot ",expectedFileRoot, filePathRoot);
116+
}
117+
}

0 commit comments

Comments
 (0)