package org.ngbw.web.actions; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.sql.SQLException; import java.util.HashMap; import java.util.Map; import java.util.Set; import org.apache.log4j.Logger; import org.apache.struts2.interceptor.validation.SkipValidation; import org.ngbw.sdk.DataDetails; import org.ngbw.sdk.Workbench; import org.ngbw.sdk.WorkbenchException; import org.ngbw.sdk.common.util.StringUtils; import org.ngbw.sdk.core.types.DataFormat; import org.ngbw.sdk.core.types.DataType; import org.ngbw.sdk.core.types.EntityType; import org.ngbw.sdk.database.CipresDataParsingScheduler; import org.ngbw.sdk.database.CipresDataParsingScheduler.CipresDataParsingSchedule; import org.ngbw.sdk.database.DataParseStatus; import org.ngbw.sdk.database.Folder; import org.ngbw.sdk.database.UserDataItem; import org.ngbw.web.controllers.FolderController; /* * 10/2012 Terri Liebowitz Schwartz * * This code, as originally written, reads the whole file into memory which * really limits the size of files we can upload. It also assumes the file is * text and tries to guess the character encoding and inserts CR/NLs, which of * course means we can't upload binary files. * * I'm going to assume files are either binary or ascii text (no fancy character * set encodings) and avoid reading the whole file into memory at once. If I * remember, my changes for this will be marked with "BIG_FILES". */ /** * Struts action class to handle creation (upload/edit/search) * of user data in the NGBW web application. * * @author Jeremy Carver */ @SuppressWarnings("serial") public class CreateData extends DataManager { private static final Logger logger = Logger.getLogger(CreateData.class.getName()); // session attribute key constants public static final String UPLOADED_FILES = "uploadedFiles"; // file upload result values public static final String UPLOAD_STATUS = "uploadStatus"; public static final String UPLOAD_SUCCESS = "SUCCESS"; public static final String UPLOAD_ERROR = "ERROR: No uploaded files were found."; public static final String UPLOAD_ERROR_PREFIX = "ERROR: "; // data upload preference keys public static final String UPLOAD_ENTITY_TYPE = "Upload_Entity_Type"; public static final String UPLOAD_DATA_TYPE = "Upload_Data_Type"; public static final String UPLOAD_DATA_FORMAT = "Upload_Data_Format"; // data item upload/edit form properties private UserDataItem dataItem; private String entityType; private String dataType; private String dataFormat; // Pasted data form control is "source" private String source = null; private String sourceLabel = null; // Uploaded file(s) info is stored here, by the form. private File[] uploads; // actual file content. private String[] uploadFileNames; // file names. public void setSourceLabel ( String s ) { sourceLabel = s; } public String getSourceLabel () { return sourceLabel; } @SkipValidation public String upload () { logger.debug("BEGIN: upload()::String"); // clear current data item and data form, since this is a new data item clearCurrentData(); clearUploadedFiles(); // pre-populate last selected upload SourceDocumentType Folder currentFolder = getCurrentFolder(); FolderController controller = getFolderController(); setEntityType(controller.getFolderPreference(currentFolder, UPLOAD_ENTITY_TYPE)); setDataType(controller.getFolderPreference(currentFolder, UPLOAD_DATA_TYPE)); setDataFormat(controller.getFolderPreference(currentFolder, UPLOAD_DATA_FORMAT)); logger.debug("END: upload()::String"); return INPUT; } public String executeUpload () { logger.debug("BEGIN: executeUpload()::String"); String retVal = (validateUpload()) ? execute() : INPUT; logger.debug("END: executeUpload()::String"); return retVal; } public String executePaste () { logger.debug("BEGIN: executePaste()::String"); String retVal = (validatePaste())? execute() : INPUT; logger.debug("END: executePaste()::String"); return retVal; } @SkipValidation @Override public String edit () { logger.info("BEGIN: edit()::String"); String retVal = INPUT; UserDataItem currentData = getCurrentData(); if (currentData != null) { setDataItem(currentData); retVal = INPUT; } else { logger.error(reportUserError("You must select a data item to edit its details.")); retVal = LIST; } logger.info("END: edit()::String"); return retVal; } @Override public String execute () { logger.info("BEGIN: execute()::String"); String retVal = INPUT; try { int saved = saveDataItems(); // TRACE - 1 if (saved < 1) { logger.error(reportUserError("No data items were saved.")); retVal = INPUT; } else { String message = saved + ((saved == 1) ? " data item" : " data items"); logger.debug(reportUserMessage(message + " successfully saved.")); if (getCurrentData() != null) { retVal = DISPLAY; } else { retVal = LIST; } } } catch ( Throwable error ) { logger.error(super.constructMessage(error, "Error saving data items")); retVal = ERROR; } logger.info("END: execute()::String"); return retVal; } @SkipValidation public String reload () { try { resolveConcepts(); return INPUT; } catch ( Throwable error ) { logger.error("Error resolving UserDataItem properties", error); return ERROR; } } @SkipValidation @Override public String cancel () { // discard input and return addActionMessage("File not saved."); UserDataItem currentData = getCurrentData(); if (getCurrentData() == null) { return LIST; } else { setDataItem(currentData); return DISPLAY; } } public UserDataItem getDataItem () { try { if (dataItem == null) { setDataItem(getWorkbenchSession().getUserDataItemInstance(getCurrentFolder())); } return dataItem; } catch ( Throwable error ) { logger.error("Error retrieving UserDataItem instance", error); return null; } } public void setDataItem ( UserDataItem dataItem ) { this.dataItem = dataItem; } @Override public String getLabel () { if (dataItem != null) { return dataItem.getLabel(); } else { return null; } } @Override public void setLabel ( String label ) { try { if (dataItem == null) { setDataItem(getWorkbenchSession().getUserDataItemInstance(getCurrentFolder())); } dataItem.setLabel(label); } catch ( Throwable error ) { logger.error("Error retrieving UserDataItem instance", error); } } @SuppressWarnings("unchecked") public Map getUploadedFiles () { return (Map) getSessionAttribute(UPLOADED_FILES); } public void setUploadedFiles ( Map files ) { if (files == null || files.size() < 1) { clearUploadedFiles(); } else { setSessionAttribute(UPLOADED_FILES, files); } } public void clearUploadedFiles () { clearSessionAttribute(UPLOADED_FILES); } public String getSource () { return source; } public void setSource ( String source ) { this.source = StringUtils.stripCarriageReturns(source); } public File[] getUpload () { return uploads; } // This function is called automatically by strut after user upload file(s) public void setUpload ( File[] uploads ) { this.uploads = uploads; } public String[] getUploadFileName () { return uploadFileNames; } public void setUploadFileName ( String[] uploadFileNames ) { this.uploadFileNames = uploadFileNames; } @Override public String getEntityType () { return entityType; } public void setEntityType ( String entityType ) { this.entityType = entityType; } @Override public String getDataType () { return dataType; } public void setDataType ( String dataType ) { this.dataType = dataType; } @Override public String getDataFormat () { return dataFormat; } public void setDataFormat ( String dataFormat ) { this.dataFormat = dataFormat; } public Map getEntityTypes () { try { return mapConceptSet(getEntityTypeSet(), "EntityType"); } catch ( Throwable error ) { logger.error("Error retrieving list of entity types", error); return null; } } public Map getDataTypes () { String entityType = getEntityType(); try { if (EntityType.get(entityType).equals(EntityType.UNKNOWN)) { return mapConceptSet(getUnknownDataTypeSet(), "DataType"); } else { return mapConceptSet( getDataTypeSet(EntityType.get(entityType)), "DataType"); } } catch ( Throwable error ) { logger.error("Error retrieving list of data types", error); return null; } } public Map getDataFormats () { String entityType = getEntityType(); String dataType = getDataType(); try { if (EntityType.get(entityType).equals(EntityType.UNKNOWN) || DataType.get(dataType).equals(DataType.UNKNOWN)) { return mapConceptSet(getUnknownDataFormatSet(), "DataFormat"); } else { return mapConceptSet( getDataFormatSet( EntityType.get(entityType), DataType.get(dataType)), "DataFormat"); } } catch ( Throwable error ) { logger.error("Error retrieving list of registered data formats for " + "specified entity type and data type", error); return null; } } protected void addUploadedFiles ( Map files ) { if (files != null && files.size() > 0) { Map uploadedFiles = getUploadedFiles(); if (uploadedFiles == null) { uploadedFiles = new HashMap(files.size()); } for (Integer index : files.keySet()) { uploadedFiles.put(index, files.get(index)); reportUserMessage("File \"" + (String) (files.get(index)[1]) + "\" added to session cache of uploaded files."); } setUploadedFiles(uploadedFiles); } } protected void saveDataPreferences () { if (resolveConcepts() == false) { return; } else { Folder currentFolder = getCurrentFolder(); FolderController controller = getFolderController(); controller.setFolderPreference(currentFolder, UPLOAD_ENTITY_TYPE, getEntityType()); controller.setFolderPreference(currentFolder, UPLOAD_DATA_TYPE, getDataType()); controller.setFolderPreference(currentFolder, UPLOAD_DATA_FORMAT, getDataFormat()); try { getWorkbenchSession().saveFolder(currentFolder); } catch ( Throwable error ) { logger.error("Error saving data upload preferences for the current folder", error); } } } protected int saveDataItems () { logger.info("BEGIN: saveDataItems()::int"); int saved = 0; // Get data pasted into "source" field of form. String userManualInputData = StringUtils.stripCarriageReturns(getSource().trim()); // If there is a paste (manually input by the user), save it. if (userManualInputData != null && !userManualInputData.isEmpty()) { String userManualInputLabel = getSourceLabel(); try { UserDataItem userDataItem = saveDataItem(userManualInputLabel, userManualInputData); if (userDataItem != null) { saved++; logger.debug(String.format("UserDataItem=[%d] saved successfully.", userDataItem.getUserDataId())); CipresDataParsingSchedule schedule = queueUserDataItemForParsing(userDataItem); if (schedule != null) { logger.debug(String.format("Successfully created data parsing schedule for UserDataID=[%d]", schedule.getUserDataId())); } else { logger.error(String.format("Failed to create data parsing schedule for UserDataID=[%d]", userDataItem.getUserDataId())); } } setCurrentData(userDataItem); } catch ( Throwable error ) { logger.error(reportUserError(error, "Error saving data item \"" + userManualInputLabel + "\"")); } } // Get files uploaded in form. File filesToBeUploaded[] = getUpload(); // file contents String fileNames[] = getUploadFileName(); // file names // If there are uploads (by themselves or in addition to pasted data), uploads will be non-null. if (filesToBeUploaded != null && filesToBeUploaded.length > 0) { for (int i = 0; i < filesToBeUploaded.length; i++) { try { // User can select multiple files to upload but we only have one label // field so we will use the filename as the label for each item and not // let the user choose. String fileName = fileNames[i]; File fileToUpload = filesToBeUploaded[i]; logger.debug(String.format("Saving file [%s] ...", fileName)); UserDataItem userDataItem = saveDataItem(fileName, fileToUpload); // TRACE - 2 if (userDataItem != null) { saved++; logger.debug(String.format("File [%s] saved successfully.", fileName)); CipresDataParsingSchedule schedule = queueUserDataItemForParsing(userDataItem); if (schedule != null) { logger.debug(String.format("Successfully created data parsing schedule for UserDataID=[%d]", schedule.getUserDataId())); } else { logger.error(String.format("Failed to create data parsing schedule for UserDataID=[%d]", userDataItem.getUserDataId())); } } } catch ( Throwable error ) { logger.error(reportUserError(error, "Error saving file \"" + fileNames[i] + "\"")); } } } refreshFolderDataTabs(); logger.info("END: saveDataItems()::int"); return saved; } protected UserDataItem saveDataItem ( String label, String data ) throws WorkbenchException { logger.info("BEGIN: saveDataItem(String, String)::UserDataItem"); Folder folder = getCurrentFolder(); // get the current folder. if (folder == null) { throw new WorkbenchException("You must select a folder to save a data item."); } else if (StringUtils.isNullOrEmpty(label, true)) { throw new WorkbenchException("You must provide a label to save a data item."); } else if (StringUtils.isNullOrEmpty(data, true)) { throw new WorkbenchException("You must provide data to save as data item."); } UserDataItem userDataItem = null; try { // retrieve data item instance userDataItem = //getWorkbenchSession().getUserDataItemInstance(folder); new UserDataItem(folder); // populate data item userDataItem.setLabel(label); userDataItem.setEntityType(EntityType.UNKNOWN); userDataItem.setDataType(DataType.UNKNOWN); /* * Terri added this code to test out the ability to guess the dataformat. I made it * conditional on the type being UNKNOWN, UNKNOWN, UNKNOWN so that it shouldn't do any * harm in ngbw, but I don't think that's what the condition should really be in CIPRES * and I'm not sure if what it should be in ngbw. But I figured this way at least we * shouldn't be erasing any useful info that the user entered. * * BUG, I think: file size in bytes isn't being displayed when we set dataformat with this * method. * * Tony: * Date: 2019-08-12 * It is not necessary to use this DataDetails class to guess the * DataFormat. We now have more reliable way of parsing data * using DataFormatParser class. * * Thus, I commented this line out, and set it to UNKNOWN. */ //userDataItem.setDataFormat(DataDetails.diagnoseFormat(data)); userDataItem.setDataFormat(DataFormat.UNKNOWN); userDataItem.setUserAssignedDataFormat(DataFormat.UNKNOWN); userDataItem.setValidated(Boolean.FALSE); userDataItem.setParsedDate(null); userDataItem.setParseStatus(DataParseStatus.UNKNOWN().getId()); userDataItem.setData(data); //userDataItem = getWorkbenchSession().saveUserDataItem(userDataItem, folder); userDataItem.setEnclosingFolder(folder); userDataItem.save(); } catch ( Throwable error ) { throw new WorkbenchException(error); } logger.info("END: saveDataItem(String, String)::UserDataItem"); return userDataItem; } /** * Saves the file content to storage folder and its name into the database. * * @param fileName * @param fileToUpload * * @return * * @throws WorkbenchException */ protected UserDataItem saveDataItem ( String fileName, File fileToUpload ) throws WorkbenchException { logger.info("BEGIN: saveDataItem(String, File)::UserDataItem"); Folder folder = getCurrentFolder(); // get current folder if (folder == null) { throw new WorkbenchException("You must select a folder to save a data item."); } else if (StringUtils.isNullOrEmpty(fileName, true)) { throw new WorkbenchException("You must provide file name to save a data item."); } else if (fileToUpload == null) { throw new WorkbenchException("You must provide a valid file to save a data item."); } FileInputStream fis = null; UserDataItem userDataItem = null; try { // retrieve data item instance userDataItem = //getWorkbenchSession().getUserDataItemInstance(folder); new UserDataItem(folder); // TRACE - 3 // populate data item Workbench.convertEncoding(fileToUpload); fis = new FileInputStream(fileToUpload); userDataItem.setData(fis); userDataItem.setLabel(fileName); userDataItem.setEntityType(EntityType.UNKNOWN); userDataItem.setDataType(DataType.UNKNOWN); userDataItem.setDataFormat(DataFormat.UNKNOWN); userDataItem.setUserAssignedDataFormat(DataFormat.UNKNOWN); userDataItem.setValidated(Boolean.FALSE); userDataItem.setParsedDate(null); userDataItem.setParseStatus(DataParseStatus.UNKNOWN().getId()); //userDataItem = getWorkbenchSession().saveUserDataItem(userDataItem, folder); userDataItem.setEnclosingFolder(folder); userDataItem.save(); } catch ( Throwable error ) { throw new WorkbenchException(error); } finally { if (fis != null) { try { fis.close(); } catch ( IOException ioe ) { logger.error(ioe.getMessage(), ioe); } } } logger.info("END: saveDataItem(String, File)::UserDataItem"); return userDataItem; } /** * @deprecated *
     * This method has been deprecated and will be removed from future versions.
     * Use {@link saveDataItem(String, String)} or {@link saveDataItem(String, File)}
     * 
* * @param label * @param data * @param fileToUpload * * @return * * @throws WorkbenchException */ // Tony: When this method is called in saveDataItems(), the user manually // input data (if any) is saved more than once if the user also uploads // file(s). Thus, I separated out into two different methods to avoid the // duplicate savings. private UserDataItem saveDataItem_tbd ( String label, String data, File fileToUpload ) throws WorkbenchException { logger.info("BEGIN: saveDataItem(String, String, File)::UserDataItem"); // get current folder Folder folder = getCurrentFolder(); if (folder == null) { throw new WorkbenchException("You must select a folder " + "to save a data item."); } else if (StringUtils.isNullOrEmpty(label, true)) { throw new WorkbenchException("You must provide a label " + "to save a data item."); } else if (fileToUpload == null && StringUtils.isNullOrEmpty(data, true)) { throw new WorkbenchException("You must provide some source data " + "to save a data item."); } // EntityType entType = EntityType.UNKNOWN; // DataType dType = DataType.UNKNOWN; /* * Terri added this code to test out the ability to guess the dataformat. I made it * conditional on the type being UNKNOWN, UNKNOWN, UNKNOWN so that it shouldn't do any * harm in ngbw, but I don't think that's what the condition should really be in CIPRES * and I'm not sure if what it should be in ngbw. But I figured this way at least we * shouldn't be erasing any useful info that the user entered. * * BUG, I think: file size in bytes isn't being displayed when we set dataformat with this * method. */ DataFormat dataFmt = null; if (fileToUpload == null) { dataFmt = DataDetails.diagnoseFormat(data); } // retrieve data item instance UserDataItem userDataItem = null; try { userDataItem = getWorkbenchSession().getUserDataItemInstance(folder); // populate data item userDataItem.setLabel(label); userDataItem.setEntityType(EntityType.UNKNOWN); userDataItem.setDataType(DataType.UNKNOWN); userDataItem.setDataFormat(dataFmt); if (fileToUpload != null) { Workbench.convertEncoding(fileToUpload); FileInputStream fis = new FileInputStream(fileToUpload); userDataItem.setData(fis); } else { userDataItem.setData(data); } userDataItem = getWorkbenchSession().saveUserDataItem(userDataItem, folder); } catch ( Throwable error ) { throw new WorkbenchException(error); } logger.info("END: saveDataItem(String, String, File)::UserDataItem"); return userDataItem; } protected boolean resolveConcepts () { String entityType = getEntityType(); String dataType = getDataType(); String dataFormat = getDataFormat(); if (entityType == null || entityType.equals("")) { setEntityType(null); setDataType(null); setDataFormat(null); return false; } else { Set dataTypes = getDataTypeSet(EntityType.get(entityType)); if (dataType == null || dataType.equals("") || dataTypes == null || dataTypes.contains(DataType.get(dataType)) == false) { setDataType(null); setDataFormat(null); return false; } else { Set dataFormats = getDataFormatSet(EntityType.get(entityType), DataType.get(dataType)); if (dataFormat == null || dataFormat.equals("") || dataFormats == null || dataFormats.contains(DataFormat.get(dataFormat)) == false) { setDataFormat(null); return false; } else { return true; } } } } protected boolean validateData () { UserDataItem dataItem = getDataItem(); if (dataItem == null) { return false; } else { String label = getLabel(); if (label == null || label.equals("")) { addFieldError("label", "Label is required."); return false; } String source = getSource(); File upload[] = getUpload(); /* * MONA: updated needed! * if ((source == null || source.equals("")) && * (upload == null || upload.canRead() == false)) * { * addActionError("You must either upload a file or enter your data directly."); * return false; * } else * { * return true; * } */ return true; } } protected boolean validateUpload () { logger.info("BEGIN: validateUpload()::boolean"); boolean status = true; Map uploadedFiles = getUploadedFiles(); if (uploadedFiles == null || uploadedFiles.isEmpty()) { logger.error(super.reportUserError("You must select one or more files to upload.")); status = false; } status &= hasFieldErrors(); logger.info("END: validateUpload()::boolean"); return status; } // Terri: don't we want to limit chars that can be used in labels? // BaseValidator.isSimpleFilename? protected boolean validatePaste () { logger.info("BEGIN: validatePaste()::boolean"); boolean status = true; String label = getSourceLabel(); String filenames[] = getUploadFileName(); if (StringUtils.isNullOrEmpty(label, true) && (filenames == null || filenames.length == 0)) { logger.error("Label missing."); addFieldError("label", "You must provide a label if you are entering your data directly."); status = false; } String src = getSource(); File[] tobeUploaded = getUpload(); if (StringUtils.isNullOrEmpty(src, true) && (tobeUploaded == null || tobeUploaded.length == 0)) { logger.error("No file is selected for upload."); addActionError("You must either upload a file or enter your data directly."); status = false; } logger.info("END: validatePaste()::boolean"); return status; } private CipresDataParsingSchedule queueUserDataItemForParsing ( UserDataItem userDataItem ) throws IOException, SQLException { if (userDataItem == null) { throw new RuntimeException("UserDataItem cannot be null."); } CipresDataParsingSchedule schedule = null; if (Workbench.getInstance().isMetaDataParserQueueEnabled(Boolean.FALSE)) { schedule = CipresDataParsingScheduler.getInstance().schedule(userDataItem); if (schedule != null) { userDataItem.setParseStatus(DataParseStatus.QUEUED().getId()); userDataItem.save(); } } return schedule; } }