Skip to content

Feature improve csv #64

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
TDevillechabrolle merged 6 commits into from
May 2, 2023
Merged

Feature improve csv #64

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
cc6c546
Resip: Make stronger pst extraction
JSLair Dec 13, 2022
6754213
Resip & Sedalib: Uncompress 7Zip files
JSLair Dec 19, 2022
3b4e145
Sedalib: improve DIP import to be used with VITAM DIP-Full
JSLair Dec 20, 2022
4e4ff02
Resip & MailExtractLib: Extract messages data object in Resip
JSLair Dec 20, 2022
84ba35e
Resip: add the csv extended format export option
JSLair Dec 22, 2022
0083bbf
Sedalib & Resip: add the csv extended format export
JSLair Dec 22, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 12 additions & 14 deletions mailextractlib/src/main/java/fr/gouv/vitam/tools/mailextractlib/core/StoreAppointment.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@
import java.security.NoSuchAlgorithmException;
import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Formatter;
import java.util.List;

Expand All @@ -19,14 +18,14 @@

public abstract class StoreAppointment extends StoreElement {

static final public int MESSAGE_STATUS_UNKNOWN=0;
static final public int MESSAGE_STATUS_LOCAL=1;
static final public int MESSAGE_STATUS_REQUEST=2;
static final public int MESSAGE_STATUS_RESPONSE_YES=3;
static final public int MESSAGE_STATUS_RESPONSE_MAY=4;
static final public int MESSAGE_STATUS_RESPONSE_NO=5;
public static final int MESSAGE_STATUS_UNKNOWN=0;
public static final int MESSAGE_STATUS_LOCAL=1;
public static final int MESSAGE_STATUS_REQUEST=2;
public static final int MESSAGE_STATUS_RESPONSE_YES=3;
public static final int MESSAGE_STATUS_RESPONSE_MAY=4;
public static final int MESSAGE_STATUS_RESPONSE_NO=5;

static final public String[] MESSAGE_STATUS_TEXT={"Unknown","Local","Request","Resp.Yes","Resp.May","Resp.No"};
public static final String[] MESSAGE_STATUS_TEXT={"Unknown","Local","Request","Resp.Yes","Resp.May","Resp.No"};

protected String uniqId;

Expand Down Expand Up @@ -87,14 +86,14 @@ public String getLogDescription() {
* @throws MailExtractLibException Any unrecoverable extraction exception (access trouble, major format problems...)
* @throws InterruptedException the interrupted exception
*/
abstract public void analyzeAppointment() throws MailExtractLibException, InterruptedException;
public abstract void analyzeAppointment() throws MailExtractLibException, InterruptedException;

/**
* Gets element name used for the csv file name construction.
*
* @return the element name
*/
static public String getElementName() {
public static String getElementName() {
return "appointments";
}

Expand All @@ -103,17 +102,15 @@ static public String getElementName() {
*
* @param ps the dedicated print stream
*/
static public void printGlobalListCSVHeader(PrintStream ps) {
public static void printGlobalListCSVHeader(PrintStream ps) {
ps.println("ID;Subject;Location;From;ToAttendees;CcAttendees;StartTime;EndTime;" +
"MiscNotes;uniqID;SequenceNumber;ModificationTime;Folder;MessageStatus;" +
"isRecurrent;RecurrencePattern;StartRecurrenceTime;EndRecurrenceTime;" +
"ExceptionFromId;ExceptionDate;isDeletion;hasAttachment");
}

/**
* Extract to the contacts list, after initialising it if first contact, and to the contact picture file.
* <p>The picture file is saved in a directory called "Contacts Pictures" and is named using the line number
* listLineId as "contact_id".ext with the extension of the original file.******************************
* Extract to the appointments list and all associated attachements and exceptions
*
* @param writeFlag the write flag
* @param father the father
Expand Down Expand Up @@ -172,6 +169,7 @@ private String normalizeUniqId(String uniqId){
result= formatter.toString();
formatter.close();
} catch (NoSuchAlgorithmException ignored) {
//ignore
}
return result;
}
Expand Down
63 changes: 35 additions & 28 deletions mailextractlib/src/main/java/fr/gouv/vitam/tools/mailextractlib/core/StoreExtractor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -169,7 +169,7 @@ public abstract class StoreExtractor {
/**
* The formatter used for zoned date conversion to String
*/
static public final DateTimeFormatter ISO_8601 = new DateTimeFormatterBuilder()
public static final DateTimeFormatter ISO_8601 = new DateTimeFormatterBuilder()
.append(ISO_LOCAL_DATE_TIME)
.optionalStart()
.appendOffsetId()
Expand All @@ -179,19 +179,24 @@ public abstract class StoreExtractor {
/**
* The map of mimetypes/scheme known relations.
*/
static HashMap<String, String> mimeTypeSchemeMap = new HashMap<String, String>();
static HashMap<String, String> mimeTypeSchemeMap = new HashMap<>();

/**
* The map of droidformat/scheme known relations.
*/
static HashMap<String, String> droidFormatSchemeMap = new HashMap<>();

/**
* The map of scheme/extractor class known relations.
*/
@SuppressWarnings("rawtypes")
static HashMap<String, Class> schemeStoreExtractorClassMap = new HashMap<String, Class>();
static HashMap<String, Class> schemeStoreExtractorClassMap = new HashMap<>();

/**
* The map of scheme/container extraction (vs single file extraction) known
* relations.
*/
static HashMap<String, Boolean> schemeContainerMap = new HashMap<String, Boolean>();
static HashMap<String, Boolean> schemeContainerMap = new HashMap<>();

/**
* Subscribes all defaults store extractor.
Expand Down Expand Up @@ -271,7 +276,8 @@ public static void initDefaultExtractors() {
private long totalRawSize;

// private field for time statistics
private Instant start, end;
private Instant start;
private Instant end;

// private object extraction root folder in store
private StoreFolder rootStoreFolder;
Expand Down Expand Up @@ -321,14 +327,26 @@ public static void initDefaultExtractors() {
* @param extractor the extractor
Copy link

@soukami soukami Apr 11, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

rajouter le nouveau paramètre à la javadoc

*/
@SuppressWarnings("rawtypes")
public static void addExtractionRelation(String mimeType, String scheme, boolean isContainer, Class extractor) {
public static void addExtractionRelation(String mimeType, String droidFormat, String scheme, boolean isContainer, Class extractor) {
// if there is a file mimetype for this scheme
if (mimeType != null)
mimeTypeSchemeMap.put(mimeType, scheme);
if (droidFormat != null)
droidFormatSchemeMap.put(droidFormat, scheme);
schemeStoreExtractorClassMap.put(scheme, extractor);
schemeContainerMap.put(scheme, isContainer);
}

/**
* Get protocol if droid format is a mail format ready for extraction.
*
* @param droidFormat the Droid format
* @return the protocol
*/
public static String getProtocolFromDroidFormat(String droidFormat) {
return droidFormatSchemeMap.get(droidFormat);
}

/**
* Compose an URL String.
*
Expand All @@ -339,7 +357,7 @@ public static void addExtractionRelation(String mimeType, String scheme, boolean
* @param path Path to the ressource
* @return the string
*/
static public String composeStoreURL(String scheme, String authority, String user, String password, String path) {
public static String composeStoreURL(String scheme, String authority, String user, String password, String path) {
String result = null;

try {
Expand Down Expand Up @@ -535,9 +553,9 @@ protected StoreExtractor(String urlString, String rootStoreFolderName, String de

this.description = ":p:" + scheme + ":u:" + user;

globalListsPSMap = new HashMap<String, PrintStream>();
elementsCounterMap = new HashMap<String, Integer>();
subElementsCounterMap = new HashMap<String, Integer>();
globalListsPSMap = new HashMap<>();
elementsCounterMap = new HashMap<>();
subElementsCounterMap = new HashMap<>();

doProgressLogIfDebug(logger,"StoreExtractor ["+this+"] created with url="+urlString+
" rootFolder="+rootStoreFolderName+" destPath="+destPathString+" rootExtractor="+rootStoreExtractor,null);
Expand Down Expand Up @@ -656,18 +674,6 @@ public long getTotalRawSize() {
return totalRawSize;
}

// /**
// * Checks for options.
// *
// * @param flags
// * constant values (CONST_...) |/+ composition
// * @return true, if successful
// */
// public boolean hasOptions(int flags) {
// return (options & flags) != 0;
// }
//

/**
* Gets the extraction context description.
*
Expand Down Expand Up @@ -902,24 +908,25 @@ else if ((path != null) && (!path.isEmpty()))
* @throws InterruptedException the interrupted exception
*/
public void listAllFolders(boolean stats) throws MailExtractLibException, InterruptedException {
String time, tmp;
String time;
String tmp;
Duration d;

Instant start = Instant.now();
start = Instant.now();

writeTargetLog();
doProgressLog(logger, MailExtractProgressLogger.GLOBAL, "mailextractlib: listing begin", null);

rootStoreFolder.listFolder(stats);

Instant end = Instant.now();
end = Instant.now();
System.out.println("--------------------------------------------------------------------------------");

d = Duration.between(start, end);
time = String.format("%dm%02ds", d.toMinutes(), d.minusMinutes(d.toMinutes()).getSeconds());
tmp = String.format("mailextractlib: terminated in %s listing %d folders", time, getElementCounter(StoreFolder.class, false));
if (stats) {
tmp += String.format(" with %d messages, for %.2f MBytes",
tmp += String.format(" with %s messages, for %.2f MBytes",
Integer.toString(getElementCounter(StoreMessage.class, false)), ((double) getTotalRawSize()) / (1024.0 * 1024.0));
}

Expand Down Expand Up @@ -974,15 +981,15 @@ public static boolean hasMagicNumber(byte[] content, byte[] magicNumber, int off
*
* @return the attachment
*/
abstract public StoreAttachment getAttachment();
public abstract StoreAttachment getAttachment();

/**
* Tests if this store extractor can generate objects lists
* (mails, contacts, appointments...).
*
* @return the flag true or false
*/
abstract public boolean canExtractObjectsLists();
public abstract boolean canExtractObjectsLists();

/**
* Gets the scheme if this content can be managed by this StoreExtractor, or
Expand Down
28 changes: 11 additions & 17 deletions ...lib/src/main/java/fr/gouv/vitam/tools/mailextractlib/store/javamail/JMStoreExtractor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,14 +57,15 @@ public class JMStoreExtractor extends StoreExtractor {
* <p>
* This is in default list.
*/
static public void subscribeStoreExtractor() {
addExtractionRelation("message/rfc822", "eml", false, JMStoreExtractor.class);
addExtractionRelation("application/mbox", "mbox", true, JMStoreExtractor.class);
addExtractionRelation(null, "thunderbird", true, JMStoreExtractor.class);
addExtractionRelation(null, "imap", true, JMStoreExtractor.class);
addExtractionRelation(null, "imaps", true, JMStoreExtractor.class);
addExtractionRelation(null, "gimap", true, JMStoreExtractor.class);
addExtractionRelation(null, "pop3", true, JMStoreExtractor.class);
public static void subscribeStoreExtractor() {
addExtractionRelation("message/rfc822", "fmt/278","eml", false, JMStoreExtractor.class);
addExtractionRelation("message/rfc822", "fmt/950","eml", false, JMStoreExtractor.class);
addExtractionRelation("application/mbox", "fmt/720","mbox", true, JMStoreExtractor.class);
addExtractionRelation(null, null,"thunderbird", true, JMStoreExtractor.class);
addExtractionRelation(null, null,"imap", true, JMStoreExtractor.class);
addExtractionRelation(null, null,"imaps", true, JMStoreExtractor.class);
addExtractionRelation(null, null,"gimap", true, JMStoreExtractor.class);
addExtractionRelation(null, null,"pop3", true, JMStoreExtractor.class);
}

// Attachment to complete with decoded form
Expand Down Expand Up @@ -209,7 +210,7 @@ public JMStoreExtractor(StoreAttachment attachment, ArchiveUnit rootNode, StoreE
}

// decode URL encoding to UTF-8
static private String getDecodedURL(String url) {
private static String getDecodedURL(String url) {
String decodedUrl = "";
try {
decodedUrl = URLDecoder.decode(url, "UTF-8");
Expand All @@ -231,8 +232,6 @@ private void setSessionProperties(Properties props) {
props.setProperty("mail.mime.parameters.strict", "false"); // not
// default
props.setProperty("mail.mime.windowsfilenames", "true"); // not default
// props.setProperty("mail.mime.ignoremultipartencoding", "false");
// //not default
}

/* (non-Javadoc)
Expand All @@ -249,11 +248,6 @@ public StoreAttachment getAttachment() {
*/
@Override
public boolean canExtractObjectsLists() {
if (this.scheme.equals("eml"))
return false;
else
return true;
return ! scheme.equals("eml");
}

;
}
8 changes: 3 additions & 5 deletions ...c/main/java/fr/gouv/vitam/tools/mailextractlib/store/microsoft/msg/MsgStoreExtractor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,9 +51,9 @@ public class MsgStoreExtractor extends StoreExtractor {
* <p>
* This is in default list.
*/
static public void subscribeStoreExtractor() {
addExtractionRelation("application/vnd.ms-outlook", "msg", false, MsgStoreExtractor.class);
addExtractionRelation(null, "msg.embeddedmsg", false, MsgStoreExtractor.class);
public static void subscribeStoreExtractor() {
addExtractionRelation("application/vnd.ms-outlook", "x-fmt/430","msg", false, MsgStoreExtractor.class);
addExtractionRelation(null, null,"msg.embeddedmsg", false, MsgStoreExtractor.class);
}

// Attachment to complete with decoded form
Expand Down Expand Up @@ -139,8 +139,6 @@ public boolean canExtractObjectsLists() {
return false;
}

;

/**
* The Constant MSG_MN.
*/
Expand Down
Loading