Skip to content

Commit

Permalink
[backend/frontend] Enable CSV Feed Ingester (#4569) (#5404)
Browse files Browse the repository at this point in the history
Co-authored-by: BocognanoSarah <sarah.bocognano@filigran.io>
Co-authored-by: Laurent Bonnet <146674147+labo-flg@users.noreply.github.com>
  • Loading branch information
3 people authored Feb 1, 2024
1 parent 4c91a7f commit 8d0da12
Show file tree
Hide file tree
Showing 41 changed files with 2,699 additions and 48 deletions.
13 changes: 11 additions & 2 deletions opencti-platform/opencti-front/lang/front/de.json
Original file line number Diff line number Diff line change
Expand Up @@ -2436,7 +2436,6 @@
"Do you want to unlink this task ?": "Möchten Sie die Verknüpfung dieser Aufgabe aufheben?",
"CREATE": "CREATE",
"entity_Analyses": "entity_Analyses",
"workflow_id": "Status",
"regardingOf": "In Bezug auf",
"WITH": "MIT",
"Pre-query to get data to be used as source entity of the relationship (limited to 5000)": "Pre-Query, um Daten zu erhalten, die als Quell-Entität der Beziehung verwendet werden sollen (begrenzt auf 5000)",
Expand Down Expand Up @@ -2501,6 +2500,13 @@
"The tag has been removed": "Das Tag wurde entfernt",
"I have read and comply with the above statement": "Ich habe die obige Erklärung gelesen und erkläre mich damit einverstanden",
"You need to activate OpenCTI enterprise edition to use this feature.": "Um diese Funktion nutzen zu können, müssen Sie die OpenCTI Enterprise Edition aktivieren.",
"workflow_id": "Status",
"Do you want to delete this CSV ingester?": "Möchten Sie diesen CSV-Importer löschen?",
"Do you want to start this CSV ingester?": "Möchten Sie diesen CSV-Importer starten?",
"Do you want to stop this CSV ingester?": "Möchten Sie diesen CSV-Importer stoppen?",
"Create a CSV Ingester": "Einen CSV-Importer erstellen",
"Update a CSV Ingester": "Einen CSV-Importer aktualisieren",
"Please, note that the test will be run on the 50 first lines": "Bitte beachten Sie, dass der Test auf den ersten 50 Zeilen durchgeführt wird",
"x_opencti_main_observable_type": "Wichtigster beobachtbarer Typ",
"Update subscription": "Update Abonnement",
"Subscribe to updates (modifications and new relations)": "Abonnement für Aktualisierungen (Änderungen und neue Beziehungen)",
Expand Down Expand Up @@ -2556,5 +2562,8 @@
"Marking definitions to use by the csv mapper...": "Markierungsdefinitionen, die vom csv-Mapper nur verwendet werden, wenn für einige Markierungsattribute die Richtlinie 'Der Benutzer darf die Markierungsdefinitionen auswählen' festgelegt ist. Andernfalls wird diese Markierungsdefinition ignoriert.",
"Option 'Let the user choose marking definitions'...": "Die Option 'Der Benutzer darf die Markierungsdefinitionen auswählen' fordert den Benutzer auf, beim Importieren der CSV-Datei auszuwählen, welche Markierungsdefinitionen angewendet werden sollen, wenn die Daten in der CSV-Datei fehlen.",
"Update without references": "Update ohne Referenzen",
"Max Confidence Level:": "Max. Konfidenzniveau:"
"Max Confidence Level:": "Max. Konfidenzniveau:",
"Please, verify the validity of the selected CSV mapper for the given URL.": "Bitte überprüfen Sie die Gültigkeit des ausgewählten CSV-Mappers für die angegebene URL.",
"Only successful tests allow the ingestion creation.": "Nur erfolgreiche Tests ermöglichen die Erstellung der Aufnahme.",
"Only successful tests allow the ingestion edition.": "Nur erfolgreiche Tests erlauben die Ingestion-Edition."
}
12 changes: 11 additions & 1 deletion opencti-platform/opencti-front/lang/front/en.json
Original file line number Diff line number Diff line change
Expand Up @@ -2447,6 +2447,12 @@
"CREATE": "CREATE",
"entity_Analyses": "entity_Analyses",
"workflow_id": "Status",
"Do you want to delete this CSV ingester?": "Do you want to delete this CSV ingester?",
"Do you want to start this CSV ingester?": "Do you want to start this CSV ingester?",
"Do you want to stop this CSV ingester?": "Do you want to stop this CSV ingester?",
"Create a CSV Ingester": "Create a CSV Ingester",
"Update a CSV Ingester": "Update a CSV Ingester",
"Please, note that the test will be run on the 50 first lines": "Please, note that the test will be run on the 50 first lines",
"regardingOf": "In regards of",
"Uploading image": "Uploading image",
"Paste": "Paste",
Expand Down Expand Up @@ -2556,5 +2562,9 @@
"Marking definitions to use by the csv mapper...": "Marking definitions used by the csv mapper only when some marking attributes have the policy set to 'Let the user choose the marking definitions'. If not this Marking definitions is ignored.",
"Option 'Let the user choose marking definitions'...": "Option 'Let the user choose marking definitions' will ask the user to choose (when importing the CSV file) which marking definitions to apply when the data is missing in the CSV file.",
"Update without references": "Update without references",
"Max Confidence Level:": "Max Confidence Level:"
"Max Confidence Level:": "Max Confidence Level:",
"Settings default values": "Default value from entity settings: {value}",
"Please, verify the validity of the selected CSV mapper for the given URL.": "Please, verify the validity of the selected CSV mapper for the given URL.",
"Only successful tests allow the ingestion creation.": "Only successful tests allow the ingestion creation.",
"Only successful tests allow the ingestion edition.": "Only successful tests allow the ingestion edition."
}
12 changes: 11 additions & 1 deletion opencti-platform/opencti-front/lang/front/es.json
Original file line number Diff line number Diff line change
Expand Up @@ -2437,6 +2437,12 @@
"entity_Analyses": "entity_Analyses",
"There are no opinions. You should create some before.": "No hay opiniones. Deberías crear algunas antes.",
"workflow_id": "Estado",
"Do you want to delete this CSV ingester?": "¿Quieres eliminar este importador CSV?",
"Do you want to start this CSV ingester?": "¿Quieres iniciar este importador CSV?",
"Do you want to stop this CSV ingester?": "¿Quieres detener este importador CSV?",
"Create a CSV Ingester": "Crear un Importador CSV",
"Update a CSV Ingester": "Actualizar un Importador CSV",
"Please, note that the test will be run on the 50 first lines": "Por favor, ten en cuenta que la prueba se ejecutará en las primeras 50 líneas",
"regardingOf": "En cuanto a",
"WITH": "CON",
"Pre-query to get data to be used as source entity of the relationship (limited to 5000)": "Consulta previa para obtener los datos que se utilizarán como entidad fuente de la relación (limitada a 5000)",
Expand Down Expand Up @@ -2556,5 +2562,9 @@
"Marking definitions to use by the csv mapper...": "Definiciones de marcado utilizadas por el mapeador csv sólo cuando algunos atributos de marcado tienen la política establecida en 'Dejar que el usuario elija las definiciones de marcado'. Si no, estas definiciones de marcado se ignoran.",
"Option 'Let the user choose marking definitions'...": "La opción 'Dejar que el usuario elija las definiciones de marcado' pedirá al usuario que elija (al importar el archivo CSV) qué definiciones de marcado aplicar cuando falten datos en el archivo CSV.",
"Update without references": "Actualizar sin referencias",
"Max Confidence Level:": "Nivel Máximo de Confianza:"
"Max Confidence Level:": "Nivel Máximo de Confianza:",
"The Max Confidence Level is currently defined at the user level. It overrides Max Confidence Level from user’s groups.": "El nivel de confianza máximo se define actualmente a nivel de usuario. Anula el nivel de confianza máximo de los grupos de usuarios.",
"Please, verify the validity of the selected CSV mapper for the given URL.": "Por favor, verifique la validez del asignador CSV seleccionado para la URL proporcionada.",
"Only successful tests allow the ingestion creation.": "Sólo las pruebas exitosas permiten la creación de la ingesta.",
"Only successful tests allow the ingestion edition.": "Sólo las pruebas exitosas permiten la edición de ingesta."
}
12 changes: 11 additions & 1 deletion opencti-platform/opencti-front/lang/front/fr.json
Original file line number Diff line number Diff line change
Expand Up @@ -2437,6 +2437,12 @@
"valid": "valide",
"There are no opinions. You should create some before.": "Il n'y a pas d'd’opinions. Vous devriez en créer avant.",
"workflow_id": "Statut",
"Do you want to delete this CSV ingester?": "Voulez-vous supprimer cet ingéreur CSV ?",
"Do you want to start this CSV ingester?": "Voulez-vous démarrer cet ingéreur CSV ?",
"Do you want to stop this CSV ingester?": "Voulez-vous arrêter cet ingéreur CSV ?",
"Create a CSV Ingester": "Créer un ingéreur CSV",
"Update a CSV Ingester": "Mettre à jour un ingéreur CSV",
"Please, note that the test will be run on the 50 first lines": "Veuillez noter que le test sera effectué sur les 50 premières lignes",
"regardingOf": "Concernant",
"WITH": "AVEC",
"Pre-query to get data to be used as source entity of the relationship (limited to 5000)": "Pré-requête pour obtenir les données à utiliser comme entité source de la relation (limité à 5000)",
Expand Down Expand Up @@ -2556,5 +2562,9 @@
"Marking definitions to use by the csv mapper...": "Les définitions de marquage ne sont utilisées par le mappeur csv que lorsque certains attributs de marquage ont la politique 'Laisser l'utilisateur choisir les définitions de marquage'. Dans le cas contraire, ces définitions de marquage sont ignorées.",
"Option 'Let the user choose marking definitions'...": "L'option 'Let the user choose marking definitions' demandera à l'utilisateur de choisir (lors de l'importation du fichier CSV) les définitions de marquage à appliquer lorsque les données sont manquantes dans le fichier CSV.",
"Update without references": "Mise à jour sans références",
"Max Confidence Level:": "Niveau de confiance maximum :"
"Max Confidence Level:": "Niveau de confiance maximum :",
"The Max Confidence Level is currently defined at the user level. It overrides Max Confidence Level from user’s groups.": "Le niveau de confiance maximum est actuellement défini au niveau de l'utilisateur. Il est prioritaire sur le niveau de confiance maximum des groupes d'utilisateurs.",
"Please, verify the validity of the selected CSV mapper for the given URL.": "Veuillez vérifier la validité du mappeur CSV sélectionné pour l'URL donnée.",
"Only successful tests allow the ingestion creation.": "Seuls les tests réussis permettent la création de l'ingestion.",
"Only successful tests allow the ingestion edition.": "Seuls les tests réussis permettent l'édition de l'ingestion."
}
12 changes: 11 additions & 1 deletion opencti-platform/opencti-front/lang/front/ja.json
Original file line number Diff line number Diff line change
Expand Up @@ -2437,6 +2437,12 @@
"entity_Analyses": "エンティティ_分析",
"There are no opinions. You should create some before.": "意見がない。事前にいくつか作っておくべきだ。",
"workflow_id": "ステータス",
"Do you want to delete this CSV ingester?": "このCSVインガスターを削除しますか?",
"Do you want to start this CSV ingester?": "このCSVインガスターを開始しますか?",
"Do you want to stop this CSV ingester?": "このCSVインガスターを停止しますか?",
"Create a CSV Ingester": "CSVインガスターを作成する",
"Update a CSV Ingester": "CSVインガスターを更新する",
"Please, note that the test will be run on the 50 first lines": "50行目までのテストを実行することに注意してください",
"regardingOf": "に関しては",
"WITH": "WITH",
"Pre-query to get data to be used as source entity of the relationship (limited to 5000)": "リレーションシップのソース・エンティティとして使用されるデータを取得するための事前クエリ(5000個まで)",
Expand Down Expand Up @@ -2556,5 +2562,9 @@
"Marking definitions to use by the csv mapper...": "いくつかのマーキング属性が'ユーザーにマーキング定義を選択させる'にポリシーが設定されている場合にのみ、csvマッパーはマーキング定義を使用します。そうでない場合、このマーキング定義は無視されます。",
"Option 'Let the user choose marking definitions'...": "オプション 'ユーザーにマーキング定義を選択させる' は、CSVファイルにデータがない場合に適用するマーキング定義を(CSVファイルのインポート時に)ユーザーに選択させます。",
"Update without references": "参照なしで更新",
"Max Confidence Level:": "最大信頼度:"
"Max Confidence Level:": "最大信頼度:",
"The Max Confidence Level is currently defined at the user level. It overrides Max Confidence Level from user’s groups.": "最大信頼度は現在ユーザーレベルで定義されています。これはユーザーグループの最大信頼度より優先されます。",
"Please, verify the validity of the selected CSV mapper for the given URL.": "指定された URL に対して選択した CSV マッパーが有効であることを確認してください。",
"Only successful tests allow the ingestion creation.": "成功したテストのみが取り込みの作成を許可します。",
"Only successful tests allow the ingestion edition.": "テストが成功した場合のみ、インジェスト エディションが許可されます。"
}
12 changes: 11 additions & 1 deletion opencti-platform/opencti-front/lang/front/zh.json
Original file line number Diff line number Diff line change
Expand Up @@ -2437,6 +2437,12 @@
"entity_Analyses": "实体_分析",
"There are no opinions. You should create some before.": "没有意见。你应该先提出一些意见。",
"workflow_id": "状态",
"Do you want to delete this CSV ingester?": "您要删除此CSV导入程序吗?",
"Do you want to start this CSV ingester?": "您要启动此CSV导入程序吗?",
"Do you want to stop this CSV ingester?": "您要停止此CSV导入程序吗?",
"Create a CSV Ingester": "创建一个CSV导入程序",
"Update a CSV Ingester": "更新一个CSV导入程序",
"Please, note that the test will be run on the 50 first lines": "请注意,测试将在前50行运行",
"regardingOf": "关于",
"WITH": "",
"Pre-query to get data to be used as source entity of the relationship (limited to 5000)": "预查询,获取用作关系源实体的数据(限制为 5000)",
Expand Down Expand Up @@ -2556,5 +2562,9 @@
"Marking definitions to use by the csv mapper...": "只有当某些标记属性的策略设置为'让用户选择标记定义'时,csv 映射器才会使用标记定义。否则,此标记定义将被忽略。",
"Option 'Let the user choose marking definitions'...": "选项 '让用户选择标记定义'将要求用户(在导入 CSV 文件时)选择在 CSV 文件中缺少数据时应用哪些标记定义。",
"Update without references": "无引用更新",
"Max Confidence Level:": "最大置信度:"
"Max Confidence Level:": "最大置信度:",
"The Max Confidence Level is currently defined at the user level. It overrides Max Confidence Level from user’s groups.": "当前最大置信水平是在用户级别定义的。它优先于用户组的最大置信度。",
"Please, verify the validity of the selected CSV mapper for the given URL.": "请验证给定 URL 所选 CSV 映射器的有效性。",
"Only successful tests allow the ingestion creation.": "只有成功的测试才允许创建摄取。",
"Only successful tests allow the ingestion edition.": "只有成功的测试才允许摄取版本。"
}
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,10 @@ const AutocompleteField = (props) => {
const fieldProps = fieldToAutocomplete(props);
delete fieldProps.helperText;
delete fieldProps.openCreate;
// Properly handle no selected option
if (fieldProps.value === '') {
fieldProps.value = null;
}
const defaultOptionToValue = (option, value) => option.value === value.value;
return (
<div style={{ position: 'relative' }}>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ const DateTimePickerField = (props) => {
onFocus: internalOnFocus,
onBlur: internalOnBlur,
error: !R.isNil(meta.error),
helperText: (!R.isNil(meta.error) && meta.error) || textFieldProps.helperText,
helperText: (!R.isNil(meta.error) && meta.error) || (textFieldProps.helperText ?? ''),
},
}}
/>
Expand Down
3 changes: 3 additions & 0 deletions opencti-platform/opencti-front/src/components/ItemIcon.jsx
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,7 @@ import {
SourcePull,
Target,
} from 'mdi-material-ui';
import TableViewIcon from '@mui/icons-material/TableView';
import { itemColor } from '../utils/Colors';

const iconSelector = (type, variant, fontSize, color, isReversed) => {
Expand Down Expand Up @@ -158,6 +159,8 @@ const iconSelector = (type, variant, fontSize, color, isReversed) => {
<AssignmentOutlined style={style} fontSize={fontSize} role="img" />
);
case 'work':
case 'csvmapper':
return <TableViewIcon style={style} fontSize={fontSize} role="img" />;
case 'connector':
return <ExtensionOutlined style={style} fontSize={fontSize} role="img" />;
case 'marking-definition':
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ const useStyles = makeStyles(() => ({
interface CreatorFieldProps {
name: string;
label: string;
isOptionEqualToValue?: (option: Option, value: string) => boolean;
onChange?: (name: string, value: Option) => void;
containerStyle?: Record<string, string | number>;
helpertext?: string;
Expand All @@ -49,6 +50,7 @@ const CreatorField: FunctionComponent<CreatorFieldProps> = ({
name,
label,
containerStyle,
isOptionEqualToValue,
onChange,
helpertext,
}) => {
Expand Down Expand Up @@ -99,6 +101,7 @@ const CreatorField: FunctionComponent<CreatorFieldProps> = ({
style={containerStyle}
noOptionsText={t_i18n('No available options')}
options={creators}
isOptionEqualToValue={isOptionEqualToValue}
onInputChange={searchCreators}
renderOption={(
props: React.HTMLAttributes<HTMLLIElement>,
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
import { Option } from '@components/common/form/ReferenceField';
import { graphql, PreloadedQuery, usePreloadedQuery } from 'react-relay';
import React, { FunctionComponent } from 'react';
import makeStyles from '@mui/styles/makeStyles';
import { Field } from 'formik';
import { CsvMapperFieldSearchQuery } from '@components/common/form/__generated__/CsvMapperFieldSearchQuery.graphql';
import { useFormatter } from '../../../../components/i18n';
import AutocompleteField from '../../../../components/AutocompleteField';
import { fieldSpacingContainerStyle } from '../../../../utils/field';
import ItemIcon from '../../../../components/ItemIcon';

const useStyles = makeStyles(() => ({
icon: {
paddingTop: 4,
display: 'inline-block',
},
text: {
display: 'inline-block',
flexGrow: 1,
marginLeft: 10,
},
autoCompleteIndicator: {
display: 'none',
},
}));

interface CsvMapperFieldComponentProps {
name: string;
isOptionEqualToValue: (option: Option, value: Option) => boolean;
onChange?: (name: string, value: Option) => void;
queryRef: PreloadedQuery<CsvMapperFieldSearchQuery>
}

export const csvMapperQuery = graphql`
query CsvMapperFieldSearchQuery($search: String) {
csvMappers(search: $search) {
edges {
node {
id
name
}
}
}
}
`;

const CsvMapperField: FunctionComponent<CsvMapperFieldComponentProps> = ({
onChange,
isOptionEqualToValue,
name,
queryRef,
}) => {
const classes = useStyles();
const { t_i18n } = useFormatter();
const data = usePreloadedQuery(csvMapperQuery, queryRef);
const csvMappersPreloaded = (data?.csvMappers?.edges || []).map(({ node }) => ({
value: node.id,
label: node.name,
}));
return (
<>
<Field
component={AutocompleteField}
style={fieldSpacingContainerStyle}
name={name}
multiple={false}
textfieldprops={{
variant: 'standard',
label: t_i18n('CSV Mappers'),
}}
noOptionsText={t_i18n('No available options')}
options={csvMappersPreloaded}
isOptionEqualToValue={isOptionEqualToValue}
onChange={onChange}
classes={{ clearIndicator: classes.autoCompleteIndicator }}
renderOption={(
props: React.HTMLAttributes<HTMLLIElement>,
option: Option,
) => (
<li {...props}>
<div className={classes.icon} style={{ color: option.color }}>
<ItemIcon type="csvmapper" />
</div>
<div className={classes.text}>{option.label}</div>
</li>
)}
/>
</>
);
};

export default CsvMapperField;
Loading

0 comments on commit 8d0da12

Please sign in to comment.