From c5ee87c9ada6d7e93fc3128931b65ba165def25c Mon Sep 17 00:00:00 2001 From: Jeremy Cloarec Date: Thu, 14 Nov 2024 11:01:27 +0100 Subject: [PATCH 1/2] [backend] modify rss http getter to a simple fetch --- .../opencti-graphql/src/manager/ingestionManager.ts | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/opencti-platform/opencti-graphql/src/manager/ingestionManager.ts b/opencti-platform/opencti-graphql/src/manager/ingestionManager.ts index fb99cdb9f550..2132491da83b 100644 --- a/opencti-platform/opencti-graphql/src/manager/ingestionManager.ts +++ b/opencti-platform/opencti-graphql/src/manager/ingestionManager.ts @@ -104,7 +104,7 @@ const pushBundleToConnectorQueue = async (context: AuthContext, ingestion: Basic // endregion // region Rss ingestion -type Getter = (uri: string) => Promise; +type Getter = (uri: string) => Promise; interface RssElement { pubDate: { _: string } @@ -159,14 +159,9 @@ const rssItemV2Convert = (turndownService: TurndownService, channel: RssElement, }; const rssHttpGetter = (): Getter => { - const httpClientOptions: GetHttpClient = { - responseType: 'text', - headers: { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:120.0) Gecko/20100101 Firefox/120.0' } - }; - const httpClient = getHttpClient(httpClientOptions); return async (uri: string) => { - const { data } = await httpClient.get(uri); - return data; + const fetchResponse = await fetch(uri); + return fetchResponse.text(); }; }; From bf2ef757e3d0c31dab02df153bbbba3ecd3e545e Mon Sep 17 00:00:00 2001 From: Jeremy Cloarec Date: Thu, 14 Nov 2024 14:49:03 +0100 Subject: [PATCH 2/2] [backend] try to retrieve item dc:date to use as pubDate if pubDate doesn't exist --- .../opencti-graphql/src/manager/ingestionManager.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/opencti-platform/opencti-graphql/src/manager/ingestionManager.ts b/opencti-platform/opencti-graphql/src/manager/ingestionManager.ts index 2132491da83b..b87f12186dc1 100644 --- a/opencti-platform/opencti-graphql/src/manager/ingestionManager.ts +++ b/opencti-platform/opencti-graphql/src/manager/ingestionManager.ts @@ -121,6 +121,7 @@ interface RssItem { 'content:encoded': { _: string } category: { _: string } | { _: string }[] pubDate: { _: string } + 'dc:date': { _: string } lastBuildDate: { _: string } updated: { _: string } } @@ -154,7 +155,7 @@ const rssItemV2Convert = (turndownService: TurndownService, channel: RssElement, link: isNotEmptyField(item.link) ? ((item.link as { _: string })._ ?? '').trim() : '', content: turndownService.turndown(item['content:encoded']?._ ?? item.content?._ ?? ''), labels: R.uniq(asArray(item.category).filter((c) => isNotEmptyField(c)).map((c) => (c as { _: string })._.trim())), - pubDate: utcDate(sanitizeForMomentParsing(item.pubDate?._ ?? pubDate?._ ?? FROM_START_STR)), + pubDate: utcDate(sanitizeForMomentParsing(item.pubDate?._ ?? item['dc:date']?._ ?? pubDate?._ ?? FROM_START_STR)), }; };