Skip to content

Commit

Permalink
Merge pull request #11017 from Recherche-Data-Gouv/9294-improvement-a…
Browse files Browse the repository at this point in the history 
…nd-internationalization-of-harvest-status

Improvement and internationalization of harvest status
  • Loading branch information
@ofahimIQSS
ofahimIQSS authored Jan 7, 2025
2 parents 825ab15 + 0c7aeb4 commit 3352bb7
Show file tree
Hide file tree
Showing 7 changed files with 76 additions and 37 deletions.
6 changes: 6 additions & 0 deletions doc/release-notes/9294-improvement-and-internationalization-of-harvest-status.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
## Improvement and internationalization of harvest status

Added a harvest status to differentiate a complete harvest with errors (Completed with failures) and without errors (Completed)
Harvest status labels are now internationalized

For more information, see issue [#9294](https://github.com/IQSS/dataverse/issues/9294)
68 changes: 40 additions & 28 deletions src/main/java/edu/harvard/iq/dataverse/harvest/client/ClientHarvestRun.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,10 @@
package edu.harvard.iq.dataverse.harvest.client;

import java.io.Serializable;
import java.util.Arrays;
import java.util.Date;

import edu.harvard.iq.dataverse.util.BundleUtil;
import jakarta.persistence.Entity;
import jakarta.persistence.GeneratedValue;
import jakarta.persistence.GenerationType;
Expand Down Expand Up @@ -40,13 +43,7 @@ public void setId(Long id) {
this.id = id;
}

public enum RunResultType { SUCCESS, FAILURE, INPROGRESS, INTERRUPTED };

private static String RESULT_LABEL_SUCCESS = "SUCCESS";
private static String RESULT_LABEL_FAILURE = "FAILED";
private static String RESULT_LABEL_INPROGRESS = "IN PROGRESS";
private static String RESULT_DELETE_IN_PROGRESS = "DELETE IN PROGRESS";
private static String RESULT_LABEL_INTERRUPTED = "INTERRUPTED";
public enum RunResultType { COMPLETED, COMPLETED_WITH_FAILURES, FAILURE, IN_PROGRESS, INTERRUPTED }

@ManyToOne
@JoinColumn(nullable = false)
Expand All @@ -68,36 +65,43 @@ public RunResultType getResult() {

public String getResultLabel() {
if (harvestingClient != null && harvestingClient.isDeleteInProgress()) {
return RESULT_DELETE_IN_PROGRESS;
return BundleUtil.getStringFromBundle("harvestclients.result.deleteInProgress");
}

if (isSuccess()) {
return RESULT_LABEL_SUCCESS;

if (isCompleted()) {
return BundleUtil.getStringFromBundle("harvestclients.result.completed");
} else if (isCompletedWithFailures()) {
return BundleUtil.getStringFromBundle("harvestclients.result.completedWithFailures");
} else if (isFailed()) {
return RESULT_LABEL_FAILURE;
return BundleUtil.getStringFromBundle("harvestclients.result.failure");
} else if (isInProgress()) {
return RESULT_LABEL_INPROGRESS;
return BundleUtil.getStringFromBundle("harvestclients.result.inProgess");
} else if (isInterrupted()) {
return RESULT_LABEL_INTERRUPTED;
return BundleUtil.getStringFromBundle("harvestclients.result.interrupted");
}
return null;
}

public String getDetailedResultLabel() {
if (harvestingClient != null && harvestingClient.isDeleteInProgress()) {
return RESULT_DELETE_IN_PROGRESS;
return BundleUtil.getStringFromBundle("harvestclients.result.deleteInProgress");
}
if (isSuccess() || isInterrupted()) {
if (isCompleted() || isCompletedWithFailures() || isInterrupted()) {
String resultLabel = getResultLabel();

resultLabel = resultLabel.concat("; "+harvestedDatasetCount+" harvested, ");
resultLabel = resultLabel.concat(deletedDatasetCount+" deleted, ");
resultLabel = resultLabel.concat(failedDatasetCount+" failed.");

String details = BundleUtil.getStringFromBundle("harvestclients.result.details", Arrays.asList(
harvestedDatasetCount.toString(),
deletedDatasetCount.toString(),
failedDatasetCount.toString()
));
if(details != null) {
resultLabel = resultLabel + "; " + details;
}
return resultLabel;
} else if (isFailed()) {
return RESULT_LABEL_FAILURE;
return BundleUtil.getStringFromBundle("harvestclients.result.failure");
} else if (isInProgress()) {
return RESULT_LABEL_INPROGRESS;
return BundleUtil.getStringFromBundle("harvestclients.result.inProgess");
}
return null;
}
Expand All @@ -106,12 +110,20 @@ public void setResult(RunResultType harvestResult) {
this.harvestResult = harvestResult;
}

public boolean isSuccess() {
return RunResultType.SUCCESS == harvestResult;
public boolean isCompleted() {
return RunResultType.COMPLETED == harvestResult;
}

public void setCompleted() {
harvestResult = RunResultType.COMPLETED;
}

public boolean isCompletedWithFailures() {
return RunResultType.COMPLETED_WITH_FAILURES == harvestResult;
}

public void setSuccess() {
harvestResult = RunResultType.SUCCESS;
public void setCompletedWithFailures() {
harvestResult = RunResultType.COMPLETED_WITH_FAILURES;
}

public boolean isFailed() {
Expand All @@ -123,12 +135,12 @@ public void setFailed() {
}

public boolean isInProgress() {
return RunResultType.INPROGRESS == harvestResult ||
return RunResultType.IN_PROGRESS == harvestResult ||
(harvestResult == null && startTime != null && finishTime == null);
}

public void setInProgress() {
harvestResult = RunResultType.INPROGRESS;
harvestResult = RunResultType.IN_PROGRESS;
}

public boolean isInterrupted() {
Expand Down
15 changes: 11 additions & 4 deletions src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,7 @@ public void doHarvest(DataverseRequest dataverseRequest, Long harvestingClientId

try {
if (harvestingClientConfig.isHarvestingNow()) {
hdLogger.log(Level.SEVERE, "Cannot start harvest, client " + harvestingClientConfig.getName() + " is already harvesting.");
hdLogger.log(Level.SEVERE, String.format("Cannot start harvest, client %s is already harvesting.", harvestingClientConfig.getName()));

} else {
harvestingClientService.resetHarvestInProgress(harvestingClientId);
Expand All @@ -176,9 +176,16 @@ public void doHarvest(DataverseRequest dataverseRequest, Long harvestingClientId
} else {
throw new IOException("Unsupported harvest type");
}
harvestingClientService.setHarvestSuccess(harvestingClientId, new Date(), harvestedDatasetIds.size(), failedIdentifiers.size(), deletedIdentifiers.size());
hdLogger.log(Level.INFO, "COMPLETED HARVEST, server=" + harvestingClientConfig.getArchiveUrl() + ", metadataPrefix=" + harvestingClientConfig.getMetadataPrefix());
hdLogger.log(Level.INFO, "Datasets created/updated: " + harvestedDatasetIds.size() + ", datasets deleted: " + deletedIdentifiers.size() + ", datasets failed: " + failedIdentifiers.size());

if (failedIdentifiers.isEmpty()) {
harvestingClientService.setHarvestCompleted(harvestingClientId, new Date(), harvestedDatasetIds.size(), failedIdentifiers.size(), deletedIdentifiers.size());
hdLogger.log(Level.INFO, String.format("\"COMPLETED HARVEST, server=%s, metadataPrefix=%s", harvestingClientConfig.getArchiveUrl(), harvestingClientConfig.getMetadataPrefix()));
} else {
harvestingClientService.setHarvestCompletedWithFailures(harvestingClientId, new Date(), harvestedDatasetIds.size(), failedIdentifiers.size(), deletedIdentifiers.size());
hdLogger.log(Level.INFO, String.format("\"COMPLETED HARVEST WITH FAILURES, server=%s, metadataPrefix=%s", harvestingClientConfig.getArchiveUrl(), harvestingClientConfig.getMetadataPrefix()));
}

hdLogger.log(Level.INFO, String.format("Datasets created/updated: %s, datasets deleted: %s, datasets failed: %s", harvestedDatasetIds.size(), deletedIdentifiers.size(), failedIdentifiers.size()));

}
} catch (StopHarvestException she) {
Expand Down
4 changes: 2 additions & 2 deletions src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -297,7 +297,7 @@ public ClientHarvestRun getLastSuccessfulRun() {
int i = harvestHistory.size() - 1;

while (i > -1) {
if (harvestHistory.get(i).isSuccess()) {
if (harvestHistory.get(i).isCompleted() || harvestHistory.get(i).isCompletedWithFailures()) {
return harvestHistory.get(i);
}
i--;
Expand All @@ -314,7 +314,7 @@ ClientHarvestRun getLastNonEmptyRun() {
int i = harvestHistory.size() - 1;

while (i > -1) {
if (harvestHistory.get(i).isSuccess()) {
if (harvestHistory.get(i).isCompleted() || harvestHistory.get(i).isCompletedWithFailures()) {
if (harvestHistory.get(i).getHarvestedDatasetCount().longValue() > 0 ||
harvestHistory.get(i).getDeletedDatasetCount().longValue() > 0) {
return harvestHistory.get(i);
Expand Down
9 changes: 7 additions & 2 deletions src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClientServiceBean.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -164,8 +164,13 @@ public void deleteClient(Long clientId) {
}

@TransactionAttribute(TransactionAttributeType.REQUIRES_NEW)
public void setHarvestSuccess(Long hcId, Date currentTime, int harvestedCount, int failedCount, int deletedCount) {
recordHarvestJobStatus(hcId, currentTime, harvestedCount, failedCount, deletedCount, ClientHarvestRun.RunResultType.SUCCESS);
public void setHarvestCompleted(Long hcId, Date currentTime, int harvestedCount, int failedCount, int deletedCount) {
recordHarvestJobStatus(hcId, currentTime, harvestedCount, failedCount, deletedCount, ClientHarvestRun.RunResultType.COMPLETED);
}

@TransactionAttribute(TransactionAttributeType.REQUIRES_NEW)
public void setHarvestCompletedWithFailures(Long hcId, Date currentTime, int harvestedCount, int failedCount, int deletedCount) {
recordHarvestJobStatus(hcId, currentTime, harvestedCount, failedCount, deletedCount, ClientHarvestRun.RunResultType.COMPLETED_WITH_FAILURES);
}

@TransactionAttribute(TransactionAttributeType.REQUIRES_NEW)
Expand Down
7 changes: 7 additions & 0 deletions src/main/java/propertyFiles/Bundle.properties
View file
Original file line number Diff line number Diff line change
Expand Up @@ -636,6 +636,13 @@ harvestclients.viewEditDialog.archiveDescription.tip=Description of the archival
harvestclients.viewEditDialog.archiveDescription.default.generic=This Dataset is harvested from our partners. Clicking the link will take you directly to the archival source of the data.
harvestclients.viewEditDialog.btn.save=Save Changes
harvestclients.newClientDialog.title.edit=Edit Group {0}
harvestclients.result.completed=Completed
harvestclients.result.completedWithFailures=Completed with failures
harvestclients.result.failure=FAILED
harvestclients.result.inProgess=IN PROGRESS
harvestclients.result.deleteInProgress=DELETE IN PROGRESS
harvestclients.result.interrupted=INTERRUPTED
harvestclients.result.details={0} harvested, {1} deleted, {2} failed.

#harvestset.xhtml
harvestserver.title=Manage Harvesting Server
Expand Down
4 changes: 3 additions & 1 deletion src/test/java/edu/harvard/iq/dataverse/api/HarvestingClientsIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -268,7 +268,7 @@ private void harvestingClientRun(boolean allowHarvestingMissingCVV) throws Inte
assertEquals("inActive", clientStatus, "Unexpected client status: "+clientStatus);

// b) Confirm that it has actually succeeded:
assertEquals("SUCCESS", responseJsonPath.getString("data.lastResult"), "Last harvest not reported a success (took "+i+" seconds)");
assertTrue(responseJsonPath.getString("data.lastResult").contains("Completed"), "Last harvest not reported a success (took "+i+" seconds)");
String harvestTimeStamp = responseJsonPath.getString("data.lastHarvest");
assertNotNull(harvestTimeStamp);

Expand All @@ -288,6 +288,8 @@ private void harvestingClientRun(boolean allowHarvestingMissingCVV) throws Inte

// Let's give the asynchronous indexing an extra sec. to finish:
Thread.sleep(1000L);
// Requires the index-harvested-metadata-source Flag feature to be enabled to search on the nickName
// Otherwise, the search must be performed with metadataSource:Harvested
Response searchHarvestedDatasets = UtilIT.search("metadataSource:" + nickName, normalUserAPIKey);
searchHarvestedDatasets.then().assertThat().statusCode(OK.getStatusCode());
searchHarvestedDatasets.prettyPrint();
Expand Down

0 comments on commit 3352bb7

Please sign in to comment.