[backend/frontend] Improve http client and proxy management (#6185)

Author: richard-julien

opencti-platform/opencti-front/src/private/components/data/ingestionCsv/IngestionCsvCreation.tsx

@@ -99,7 +99,6 @@ const IngestionCsvCreation: FunctionComponent<IngestionCsvCreationProps> = ({ pa
     } else if (values.authentication_type === 'certificate') {
       authenticationValue = `${values.cert}:${values.key}:${values.ca}`;
     }
-    const userId = typeof values.user_id === 'string' ? values.user_id : values.user_id.value;
     const input = {
       name: values.name,
       description: values.description,
@@ -108,7 +107,7 @@ const IngestionCsvCreation: FunctionComponent<IngestionCsvCreationProps> = ({ pa
       authentication_type: values.authentication_type,
       authentication_value: authenticationValue,
       current_state_date: values.current_state_date,
-      user_id: userId,
+      user_id: typeof values.user_id === 'string' ? values.user_id : values.user_id.value,
     };
     commit({
       variables: {
@@ -330,8 +329,7 @@ const IngestionCsvCreation: FunctionComponent<IngestionCsvCreationProps> = ({ pa
               <IngestionCsvMapperTestDialog
                 open={open}
                 onClose={() => setOpen(false)}
-                uri={values.uri}
-                csvMapperId={values.csv_mapper_id}
+                values={values}
                 setIsCreateDisabled={setIsCreateDisabled}
               />
             </Form>

opencti-platform/opencti-front/src/private/components/data/ingestionCsv/IngestionCsvEdition.tsx

@@ -361,8 +361,7 @@ const IngestionCsvEdition: FunctionComponent<IngestionCsvEditionProps> = ({
           <IngestionCsvMapperTestDialog
             open={open}
             onClose={() => setOpen(false)}
-            uri={values.uri}
-            csvMapperId={values.csv_mapper_id}
+            values={values}
           />
         </Form>
       )}

opencti-platform/opencti-front/src/private/components/data/ingestionCsv/IngestionCsvMapperTestDialog.tsx

@@ -1,22 +1,21 @@
-import { graphql } from 'react-relay';
+import { graphql, useMutation } from 'react-relay';
 import React, { FunctionComponent, useState } from 'react';
 import Dialog from '@mui/material/Dialog';
 import DialogTitle from '@mui/material/DialogTitle';
 import DialogContent from '@mui/material/DialogContent';
 import Button from '@mui/material/Button';
 import Box from '@mui/material/Box';
 import CodeBlock from '@components/common/CodeBlock';
-import { IngestionCsvMapperTestDialogQuery$data } from '@components/data/ingestionCsv/__generated__/IngestionCsvMapperTestDialogQuery.graphql';
-import { Option } from '@components/common/form/ReferenceField';
-import TextField from '@mui/material/TextField';
 import Alert from '@mui/material/Alert';
+import { Option } from '@components/common/form/ReferenceField';
+import { IngestionCsvMapperTestDialogMutation$data } from '@components/data/ingestionCsv/__generated__/IngestionCsvMapperTestDialogMutation.graphql';
 import Loader, { LoaderVariant } from '../../../../components/Loader';
 import { useFormatter } from '../../../../components/i18n';
-import { fetchQuery, handleError } from '../../../../relay/environment';
+import { handleError } from '../../../../relay/environment';
 
-const ingestionCsvMapperTestQuery = graphql`
-  query IngestionCsvMapperTestDialogQuery($uri: String!, $csv_mapper_id: String!) {
-    test_mapper(uri: $uri, csv_mapper_id: $csv_mapper_id) {
+const ingestionCsvMapperTestMutation = graphql`
+  mutation IngestionCsvMapperTestDialogMutation($input: IngestionCsvAddInput!) {
+    ingestionCsvTester(input: $input) {
       nbEntities
       nbRelationships
       objects
@@ -27,76 +26,73 @@ const ingestionCsvMapperTestQuery = graphql`
 interface IngestionCsvMapperTestDialogProps {
   open: boolean
   onClose: () => void
-  uri: string
-  csvMapperId: string | Option
+  values: {
+    name: string,
+    description?: string | null,
+    authentication_type: string,
+    authentication_value?: string | null,
+    current_state_date: Date | null,
+    uri: string,
+    ingestion_running?: boolean | null,
+    csv_mapper_id: string | Option,
+    user_id: string | Option
+  }
   setIsCreateDisabled?: React.Dispatch<React.SetStateAction<boolean>>
 }
 
 const IngestionCsvMapperTestDialog: FunctionComponent<IngestionCsvMapperTestDialogProps> = ({
   open,
   onClose,
-  uri,
-  csvMapperId,
+  values,
   setIsCreateDisabled,
 }) => {
   const { t_i18n } = useFormatter();
-  const [result, setResult] = useState<IngestionCsvMapperTestDialogQuery$data | undefined>(undefined);
+  const [result, setResult] = useState<IngestionCsvMapperTestDialogMutation$data | undefined>(undefined);
+  const [commitTest] = useMutation(ingestionCsvMapperTestMutation);
   const [loading, setLoading] = useState<boolean>(false);
 
   const handleClose = () => {
     setResult(undefined);
     onClose();
   };
 
-  const onTest = (url: string, csv_mapper_id: string) => {
+  const onTest = () => {
     setLoading(true);
-    fetchQuery(ingestionCsvMapperTestQuery, { uri: url, csv_mapper_id })
-      .toPromise()
-      .then((data) => {
-        const resultTest = (data as IngestionCsvMapperTestDialogQuery$data)
-          .test_mapper;
+    commitTest({
+      variables: {
+        input: {
+          name: values.name,
+          description: values.description,
+          authentication_type: values.authentication_type,
+          authentication_value: values.authentication_value,
+          current_state_date: values.current_state_date,
+          uri: values.uri,
+          ingestion_running: values.ingestion_running,
+          user_id: typeof values.user_id === 'string' ? values.user_id : values.user_id.value,
+          csv_mapper_id: typeof values.csv_mapper_id === 'string' ? values.csv_mapper_id : values.csv_mapper_id.value,
+        },
+      },
+      onCompleted: (data) => {
+        const resultTest = (data as IngestionCsvMapperTestDialogMutation$data);
         if (resultTest) {
-          setResult({
-            test_mapper: {
-              ...resultTest,
-            },
-          });
+          setResult(resultTest);
           if (setIsCreateDisabled) {
-            setIsCreateDisabled(resultTest.nbEntities === 0);
+            setIsCreateDisabled(resultTest.ingestionCsvTester?.nbEntities === 0);
           }
         }
         setLoading(false);
-      }).catch((error) => {
+      },
+      onError: (error) => {
         handleError(error);
         setLoading(false);
-      });
+      },
+    });
   };
 
   return (
     <Dialog open={open} onClose={handleClose} PaperProps={{ elevation: 1 }}>
       <DialogTitle>{t_i18n('Testing csv mapper')}</DialogTitle>
       <DialogContent>
-        <Box
-          sx={{ marginBottom: '12px' }}
-        >
-          <TextField
-            label="CSV feed URL"
-            defaultValue={uri}
-            InputProps={{
-              readOnly: true,
-            }}
-            fullWidth
-            sx={{ marginBottom: '12px' }}
-          />
-          <TextField
-            label="CSV mapper"
-            defaultValue={typeof csvMapperId === 'string' ? csvMapperId : csvMapperId.label}
-            InputProps={{
-              readOnly: true,
-            }}
-            fullWidth
-          />
-        </Box>
         <Box>
           <div style={{ width: '100%', marginTop: 10 }}>
             <Alert
@@ -113,8 +109,8 @@ const IngestionCsvMapperTestDialog: FunctionComponent<IngestionCsvMapperTestDial
         >
           <Button
             variant="contained"
-            color={result?.test_mapper?.nbEntities ? 'primary' : 'secondary'}
-            onClick={() => onTest(uri, typeof csvMapperId === 'string' ? csvMapperId : csvMapperId.value)}
+            color={result?.ingestionCsvTester?.nbEntities ? 'primary' : 'secondary'}
+            onClick={() => onTest()}
           >
             {t_i18n('Test')}
           </Button>
@@ -135,14 +131,14 @@ const IngestionCsvMapperTestDialog: FunctionComponent<IngestionCsvMapperTestDial
               }}
                >
               <span>{t_i18n('Objects found')} : </span>
-              <span><strong>{result?.test_mapper?.nbEntities} </strong> {t_i18n('Entities')}</span>
-              <span><strong>{result?.test_mapper?.nbRelationships}</strong> {t_i18n('Relationships')}</span>
+              <span><strong>{result?.ingestionCsvTester?.nbEntities} </strong> {t_i18n('Entities')}</span>
+              <span><strong>{result?.ingestionCsvTester?.nbRelationships}</strong> {t_i18n('Relationships')}</span>
             </Box>
           }
         </Box>
         <Box sx={{ marginTop: '8px' }}>
           <CodeBlock
-            code={result?.test_mapper?.objects || t_i18n('You will find here the result in JSON format.')}
+            code={result?.ingestionCsvTester?.objects || t_i18n('You will find here the result in JSON format.')}
             language={'json'}
           />
         </Box>

opencti-platform/opencti-front/src/private/components/data/ingestionTaxii/IngestionTaxiiLine.jsx

@@ -107,7 +107,7 @@ class IngestionTaxiiLineLineComponent extends Component {
                 className={classes.bodyItem}
                 style={{ width: dataColumns.added_after_start.width }}
               >
-                <code>{node.added_after_start}</code>
+                <code>{node.added_after_start || node.current_state_cursor}</code>
               </div>
             </div>
           }
@@ -144,6 +144,7 @@ const IngestionTaxiiLineFragment = createFragmentContainer(
         version
         ingestion_running
         added_after_start
+        current_state_cursor
       }
     `,
   },

opencti-platform/opencti-front/src/schema/relay.schema.graphql

@@ -7375,7 +7375,6 @@ type Query {
   ingestionTaxiis(first: Int, after: ID, orderBy: IngestionTaxiiOrdering, orderMode: OrderingMode, filters: FilterGroup, includeAuthorities: Boolean, search: String): IngestionTaxiiConnection
   ingestionCsv(id: String!): IngestionCsv
   ingestionCsvs(first: Int, after: ID, orderBy: IngestionCsvOrdering, orderMode: OrderingMode, filters: FilterGroup, includeAuthorities: Boolean, search: String): IngestionCsvConnection
-  test_mapper(uri: String!, csv_mapper_id: String!): CsvMapperTestResult
   indicator(id: String!): Indicator
   indicators(first: Int, after: ID, orderBy: IndicatorsOrdering, orderMode: OrderingMode, filters: FilterGroup, search: String): IndicatorConnection
   indicatorsTimeSeries(objectId: String, field: String!, operation: StatsOperation!, startDate: DateTime!, endDate: DateTime!, interval: String!, filters: FilterGroup): [TimeSeries]
@@ -8125,6 +8124,7 @@ type Mutation {
   ingestionTaxiiAdd(input: IngestionTaxiiAddInput!): IngestionTaxii
   ingestionTaxiiDelete(id: ID!): ID
   ingestionTaxiiFieldPatch(id: ID!, input: [EditInput!]!): IngestionTaxii
+  ingestionCsvTester(input: IngestionCsvAddInput!): CsvMapperTestResult
   ingestionCsvAdd(input: IngestionCsvAddInput!): IngestionCsv
   ingestionCsvDelete(id: ID!): ID
   ingestionCsvFieldPatch(id: ID!, input: [EditInput!]!): IngestionCsv
@@ -10593,6 +10593,7 @@ type IngestionTaxii implements InternalObject & BasicObject {
   authentication_value: String
   user_id: String
   user: Creator
+  current_state_cursor: String
   added_after_start: DateTime
   ingestion_running: Boolean
 }

opencti-platform/opencti-graphql/src/database/rabbitmq.js

@@ -103,15 +103,16 @@ export const metrics = async (context, user) => {
   const metricApi = async () => {
     const ssl = USE_SSL_MGMT ? 's' : '';
     const baseURL = `http${ssl}://${HOSTNAME_MGMT}:${PORT_MGMT}`;
-    const httpClient = getHttpClient({
+    const httpClientOptions = {
       baseURL,
       responseType: 'json',
       rejectUnauthorized: RABBITMQ_MGMT_REJECT_UNAUTHORIZED,
       auth: {
         username: USERNAME,
         password: PASSWORD,
       },
-    });
+    };
+    const httpClient = getHttpClient(httpClientOptions);
     const overview = await httpClient.get('/api/overview').then((response) => response.data);
     const queues = await httpClient.get(`/api/queues${VHOST_PATH}`).then((response) => response.data);
     // Compute number of push queues

opencti-platform/opencti-graphql/src/domain/connector.js

@@ -207,7 +207,7 @@ export const fetchRemoteStreams = async (context, user, { uri, token, ssl_verify
     const httpClientOptions = { headers, rejectUnauthorized: ssl_verify ?? false, responseType: 'json' };
     const httpClient = getHttpClient(httpClientOptions);
     const remoteUri = `${uri.endsWith('/') ? uri.slice(0, -1) : uri}/graphql`;
-    const { data } = await httpClient.post(remoteUri, { query }, { withCredentials: true });
+    const { data } = await httpClient.post(remoteUri, { query });
     return data.data.streamCollections.edges.map((e) => e.node);
   } catch (e) {
     throw ValidationError('uri', { message: 'Error getting the streams from remote OpenCTI', cause: e });

opencti-platform/opencti-graphql/src/generated/graphql.ts

@@ -9911,6 +9911,7 @@ export type IngestionTaxii = BasicObject & InternalObject & {
   authentication_value?: Maybe<Scalars['String']['output']>;
   collection: Scalars['String']['output'];
   created_at?: Maybe<Scalars['DateTime']['output']>;
+  current_state_cursor?: Maybe<Scalars['String']['output']>;
   description?: Maybe<Scalars['String']['output']>;
   entity_type: Scalars['String']['output'];
   id: Scalars['ID']['output'];
@@ -12264,6 +12265,7 @@ export type Mutation = {
   ingestionCsvAdd?: Maybe<IngestionCsv>;
   ingestionCsvDelete?: Maybe<Scalars['ID']['output']>;
   ingestionCsvFieldPatch?: Maybe<IngestionCsv>;
+  ingestionCsvTester?: Maybe<CsvMapperTestResult>;
   ingestionRssAdd?: Maybe<IngestionRss>;
   ingestionRssDelete?: Maybe<Scalars['ID']['output']>;
   ingestionRssFieldPatch?: Maybe<IngestionRss>;
@@ -13194,6 +13196,11 @@ export type MutationIngestionCsvFieldPatchArgs = {
 };
 
 
+export type MutationIngestionCsvTesterArgs = {
+  input: IngestionCsvAddInput;
+};
+
+
 export type MutationIngestionRssAddArgs = {
   input: IngestionRssAddInput;
 };
@@ -17692,7 +17699,6 @@ export type Query = {
   tasks?: Maybe<TaskConnection>;
   taxiiCollection?: Maybe<TaxiiCollection>;
   taxiiCollections?: Maybe<TaxiiCollectionConnection>;
-  test_mapper?: Maybe<CsvMapperTestResult>;
   threatActor?: Maybe<ThreatActor>;
   threatActorGroup?: Maybe<ThreatActorGroup>;
   threatActorIndividual?: Maybe<ThreatActorIndividual>;
@@ -19943,12 +19949,6 @@ export type QueryTaxiiCollectionsArgs = {
 };
 
 
-export type QueryTest_MapperArgs = {
-  csv_mapper_id: Scalars['String']['input'];
-  uri: Scalars['String']['input'];
-};
-
-
 export type QueryThreatActorArgs = {
   id?: InputMaybe<Scalars['String']['input']>;
 };
@@ -32600,6 +32600,7 @@ export type IngestionTaxiiResolvers<ContextType = any, ParentType extends Resolv
   authentication_value?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
   collection?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
   created_at?: Resolver<Maybe<ResolversTypes['DateTime']>, ParentType, ContextType>;
+  current_state_cursor?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
   description?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
   entity_type?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
   id?: Resolver<ResolversTypes['ID'], ParentType, ContextType>;
@@ -33587,6 +33588,7 @@ export type MutationResolvers<ContextType = any, ParentType extends ResolversPar
   ingestionCsvAdd?: Resolver<Maybe<ResolversTypes['IngestionCsv']>, ParentType, ContextType, RequireFields<MutationIngestionCsvAddArgs, 'input'>>;
   ingestionCsvDelete?: Resolver<Maybe<ResolversTypes['ID']>, ParentType, ContextType, RequireFields<MutationIngestionCsvDeleteArgs, 'id'>>;
   ingestionCsvFieldPatch?: Resolver<Maybe<ResolversTypes['IngestionCsv']>, ParentType, ContextType, RequireFields<MutationIngestionCsvFieldPatchArgs, 'id' | 'input'>>;
+  ingestionCsvTester?: Resolver<Maybe<ResolversTypes['CsvMapperTestResult']>, ParentType, ContextType, RequireFields<MutationIngestionCsvTesterArgs, 'input'>>;
   ingestionRssAdd?: Resolver<Maybe<ResolversTypes['IngestionRss']>, ParentType, ContextType, RequireFields<MutationIngestionRssAddArgs, 'input'>>;
   ingestionRssDelete?: Resolver<Maybe<ResolversTypes['ID']>, ParentType, ContextType, RequireFields<MutationIngestionRssDeleteArgs, 'id'>>;
   ingestionRssFieldPatch?: Resolver<Maybe<ResolversTypes['IngestionRss']>, ParentType, ContextType, RequireFields<MutationIngestionRssFieldPatchArgs, 'id' | 'input'>>;
@@ -35082,7 +35084,6 @@ export type QueryResolvers<ContextType = any, ParentType extends ResolversParent
   tasks?: Resolver<Maybe<ResolversTypes['TaskConnection']>, ParentType, ContextType, Partial<QueryTasksArgs>>;
   taxiiCollection?: Resolver<Maybe<ResolversTypes['TaxiiCollection']>, ParentType, ContextType, RequireFields<QueryTaxiiCollectionArgs, 'id'>>;
   taxiiCollections?: Resolver<Maybe<ResolversTypes['TaxiiCollectionConnection']>, ParentType, ContextType, Partial<QueryTaxiiCollectionsArgs>>;
-  test_mapper?: Resolver<Maybe<ResolversTypes['CsvMapperTestResult']>, ParentType, ContextType, RequireFields<QueryTest_MapperArgs, 'csv_mapper_id' | 'uri'>>;
   threatActor?: Resolver<Maybe<ResolversTypes['ThreatActor']>, ParentType, ContextType, Partial<QueryThreatActorArgs>>;
   threatActorGroup?: Resolver<Maybe<ResolversTypes['ThreatActorGroup']>, ParentType, ContextType, Partial<QueryThreatActorGroupArgs>>;
   threatActorIndividual?: Resolver<Maybe<ResolversTypes['ThreatActorIndividual']>, ParentType, ContextType, RequireFields<QueryThreatActorIndividualArgs, 'id'>>;