From d14c1e71b5060485f9281e81f91f6a736724c2e3 Mon Sep 17 00:00:00 2001 From: Dale Wahl Date: Fri, 22 Nov 2024 13:46:49 +0100 Subject: [PATCH 01/54] fix missing card(s) from douyin search --- modules/douyin.js | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/modules/douyin.js b/modules/douyin.js index 7bcc2ac..6c52cb4 100644 --- a/modules/douyin.js +++ b/modules/douyin.js @@ -180,7 +180,7 @@ zeeschuimer.register_module( // Collection of videos let mix_videos = search_result["aweme_mix_info"]["mix_items"]; let first_mix_vid = true; - for(let j in mix_videos) { + for (let j in mix_videos) { // Each video has mix_info data // item_data["mix_info"]["statis"]["current_episode"] is an int starting at 1 representing the video order let item_data = mix_videos[j]; @@ -198,6 +198,16 @@ zeeschuimer.register_module( mix_video_count++; } mix_count++; + } else if ("card_info" in search_result && "attached_info" in search_result["card_info"] && "aweme_list" in search_result["card_info"]["attached_info"]) { + // Seen card_unique_name: douyin_playlet_v1 + let first = true; + for (let i in search_result["card_info"]["attached_info"]["aweme_list"]) { + let item_data = search_result["card_info"]["attached_info"]["aweme_list"][i]; + item_data["id"] = item_data["aweme_id"]; + usable_items.push(item_data); + } + // I have only seen these with 1 video, but... ? + console.log(`Collected ${usable_items.length} Douyin videos for ${search_result["card_unique_name"]}`) } else if (["baike_wiki_doc", "douyin_trending"].includes(search_result["card_unique_name"])) { // baike_wiki_doc are cool chinese wiki cards; I have seen them explaining the search term used // douyin_trending trending data @@ -216,17 +226,17 @@ zeeschuimer.register_module( } if (!(usable_items.length === 0)) { // Return the usable items; logging to console to compare with what is displayed on the page - // let usable_count = 0; - // for (let i in usable_items) { - // usable_count++; - // let item = usable_items[i]; - // if ('desc' in item && item['desc']) { - // // streams' desc are $undefined - // console.log(` Item ${i}: ${item['desc']}`); - // } else { - // console.log(`Item ${i} has no description`); - // } - // } + let usable_count = 0; + for (let i in usable_items) { + usable_count++; + let item = usable_items[i]; + if ('desc' in item && item['desc']) { + // streams' desc are $undefined + console.log(` Item ${i}: ${item['desc']}`); + } else { + console.log(`Item ${i} has no description`); + } + } console.log(`Found ${usable_items.length} Douyin videos on page ${source_platform_url}`) return usable_items; } else { From 7c9eba12120aca96fae1e6fd00cdeffcd635a622 Mon Sep 17 00:00:00 2001 From: Dale Wahl Date: Fri, 22 Nov 2024 13:54:05 +0100 Subject: [PATCH 02/54] update test for douyin --- tests/tests.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/tests.json b/tests/tests.json index e9c756b..caca552 100644 --- a/tests/tests.json +++ b/tests/tests.json @@ -262,7 +262,7 @@ "douyin.com": { "Front Page": { "https://www.douyin.com/discover": { - "expected": 20, + "expected": 70, "more-after-scroll": true, "wait": 10, "comment": "Douyin loves captchas; longer wait is needed until for them to load", @@ -279,7 +279,7 @@ }, "User page": { "https://www.douyin.com/user/MS4wLjABAAAAqtpQNZmjRtGJdpyk0gbAh6OpqqBjJddNTgLhmK3Y0CI": { - "expected": 21, + "expected": 20, "more-after-scroll": false, "wait": 10, "comment": "more-after-scroll is true if logged in, false if not", @@ -288,7 +288,7 @@ }, "Search": { "https://www.douyin.com/search/%E6%90%9E%E7%AC%91": { - "expected": 9, + "expected": 26, "more-after-scroll": true, "wait": 10, "captcha-selector": "div#captcha_container" @@ -304,7 +304,7 @@ }, "Channels": { "https://www.douyin.com/channel/300205": { - "expected": 20, + "expected": 30, "more-after-scroll": true, "wait": 10, "captcha-selector": "div#captcha_container" @@ -312,7 +312,7 @@ }, "Individual post page": { "https://www.douyin.com/?modal_id=7092325988377316616": { - "expected": 10, + "expected": 2, "more-after-scroll": false, "wait": 10, "comment": "Douyin loads additional videos ready to scroll to even on individual post pages", From 6adb0c97a972b29cef728f6aa81fac5a1d7b2273 Mon Sep 17 00:00:00 2001 From: Dale Wahl Date: Wed, 27 Nov 2024 14:22:41 +0100 Subject: [PATCH 03/54] douyin: skip embedded videos if ?modal_is in source URL (only displaying one video); collect individual videos from /video/ links --- modules/douyin.js | 87 +++++++++++++++++++++++++++-------------------- 1 file changed, 50 insertions(+), 37 deletions(-) diff --git a/modules/douyin.js b/modules/douyin.js index 6c52cb4..9850e7a 100644 --- a/modules/douyin.js +++ b/modules/douyin.js @@ -52,7 +52,7 @@ zeeschuimer.register_module( return []; } try { - // Extract second JSON and search for approporiate dictionary + // Extract second JSON and search for appropriate dictionary let parsed_list = JSON.parse(temp_data[1].substring(temp_data[1].indexOf(":[") + 1)) for (let j = 1; j < parsed_list.length; j++) { response = parsed_list[j]; @@ -90,28 +90,34 @@ zeeschuimer.register_module( let usable_items = []; if (from_embed) { - // Embedded data - if ("value" in data) { - // Two places where we can find videos (at least...) + if (source_url.includes("?modal_id=")) { + // This is an individual video page and the embedded data is NOT the video itself! Only visible when the individual video is closed. + console.log("Embedded videos on individual page do not contain displayed video") + console.log(data) + } else { + // Embedded data let awemeList_count = 0; - if (("homeFetchData" in data["value"]) && !(data["value"]["homeFetchData"] === "$undefined") && ("awemeList" in data["value"]["homeFetchData"])) { - for (let i in data["value"]["homeFetchData"]["awemeList"]) { - let item = data["value"]["homeFetchData"]["awemeList"][i]; - item["id"] = item["awemeId"]; - item["ZS_collected_from_embed"] = from_embed; - usable_items.push(item); - } - awemeList_count = data["value"]["homeFetchData"]["awemeList"].length; - } let recommendAwemeList_count = 0; - if ("recommendAwemeList" in data["value"]) { - for (let i in data["value"]["recommendAwemeList"]) { - let item = data["value"]["recommendAwemeList"][i]; - item["id"] = item["awemeId"]; - item["ZS_collected_from_embed"] = from_embed; - usable_items.push(item); + if ("value" in data) { + // Two places where we can find videos (at least...) + if (("homeFetchData" in data["value"]) && !(data["value"]["homeFetchData"] === "$undefined") && ("awemeList" in data["value"]["homeFetchData"])) { + for (let i in data["value"]["homeFetchData"]["awemeList"]) { + let item = data["value"]["homeFetchData"]["awemeList"][i]; + item["id"] = item["awemeId"]; + item["ZS_collected_from_embed"] = from_embed; + usable_items.push(item); + } + awemeList_count = data["value"]["homeFetchData"]["awemeList"].length; + } + if ("recommendAwemeList" in data["value"]) { + for (let i in data["value"]["recommendAwemeList"]) { + let item = data["value"]["recommendAwemeList"][i]; + item["id"] = item["awemeId"]; + item["ZS_collected_from_embed"] = from_embed; + usable_items.push(item); + } + recommendAwemeList_count = data["value"]["recommendAwemeList"].length; } - recommendAwemeList_count = data["value"]["recommendAwemeList"].length; } if (usable_items.length === 0) { @@ -121,7 +127,12 @@ zeeschuimer.register_module( console.log(`Collected ${usable_items.length} Douyin videos from embedded HTML (awemeList ${awemeList_count}, recommendAwemeList ${recommendAwemeList_count})`) } } - + } else if ("aweme_detail" in data) { + // Single video on page (e.g. www.douyin.com/video/7092325988377316616 + let item = data["aweme_detail"]; + item["id"] = item["aweme_id"]; + usable_items.push(item); + console.log(`Collected single video from ${source_platform_url}`) } else if ("cards" in data) { // Front Page (首页) tab (i.e. douyin.com/discover) if (source_platform_url.includes("/discover")) { @@ -147,8 +158,10 @@ zeeschuimer.register_module( // Recommend (e.g. home page douyin.com or douyin.com/?recommend=1 etc.) page tab loads multiple aweme_list objects, but only one is visible let url = new URL(source_platform_url); - if ((url.pathname === '/' || url.pathname === '') && (["locate_item_available", "chime_video_list"].some(function(e){ return e in data;}))) { + if ( source_platform_url.includes("?modal_id=") || source_platform_url.includes("/video/") || ((url.pathname === '/' || url.pathname === '') && (["locate_item_available", "chime_video_list"].some(function(e){ return e in data;}))) ) { // These are not visible though they may appear when navigating to the Front Page and possibly elsewhere + // console.log(`Collected but not visible Douyin videos from ${source_platform_url}`) + // console.log(data) } else { for (let i in data["aweme_list"]) { let item_data = data["aweme_list"][i]; @@ -217,8 +230,8 @@ zeeschuimer.register_module( } } console.log(`Collected ${videos_count} Douyin videos and ${mix_count} mixes (containing ${mix_video_count} videos)`) - } else if ((("e" in data) && ("sc" in data) && ("tc" in data) && (3 === Object.keys(data).length)) || ("StabilityStatistics" in data)) { - // These appear to be status pings of some kind + } else if ((("e" in data) && ("sc" in data) && ("tc" in data) && (3 === Object.keys(data).length)) || ("StabilityStatistics" in data) || "maigc" in data || ("e" in data && "sc" in data && Object.keys(data).length === 2)) { + // These appear to be status pings and other non-video data return []; } else { // console.log("MAYBE INTERESTING") @@ -226,21 +239,21 @@ zeeschuimer.register_module( } if (!(usable_items.length === 0)) { // Return the usable items; logging to console to compare with what is displayed on the page - let usable_count = 0; - for (let i in usable_items) { - usable_count++; - let item = usable_items[i]; - if ('desc' in item && item['desc']) { - // streams' desc are $undefined - console.log(` Item ${i}: ${item['desc']}`); - } else { - console.log(`Item ${i} has no description`); - } - } - console.log(`Found ${usable_items.length} Douyin videos on page ${source_platform_url}`) + // let usable_count = 0; + // for (let i in usable_items) { + // usable_count++; + // let item = usable_items[i]; + // if ('desc' in item && item['desc']) { + // // streams' desc are $undefined + // console.log(` Item ${i}: ${item['id']} - ${item['desc']}`); + // } else { + // console.log(`Item ${i}: ${item['id']} - no description`); + // } + // } + // console.log(`Found ${usable_items.length} Douyin videos on page ${source_platform_url}`) return usable_items; } else { - //console.log("Detected expected object by no usable items found") + // console.log("Detected expected object by no usable items found") } // if () { From 6372e0a47c675e03a7a5c8b3584b475879b9a656 Mon Sep 17 00:00:00 2001 From: Dale Wahl Date: Wed, 27 Nov 2024 14:25:50 +0100 Subject: [PATCH 04/54] add douyin test --- tests/tests.json | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/tests/tests.json b/tests/tests.json index caca552..1265483 100644 --- a/tests/tests.json +++ b/tests/tests.json @@ -312,10 +312,19 @@ }, "Individual post page": { "https://www.douyin.com/?modal_id=7092325988377316616": { - "expected": 2, + "expected": 1, + "more-after-scroll": false, + "wait": 10, + "comment": "Douyin loads full background page (i.e. www.douyin.com) and then individual post in popup", + "captcha-selector": "div#captcha_container" + } + }, + "Individual post page v2": { + "https://www.douyin.com/video/7416052349312961819": { + "expected": 1, "more-after-scroll": false, "wait": 10, - "comment": "Douyin loads additional videos ready to scroll to even on individual post pages", + "comment": "Douyin also loads additional videos in recommendation bar to side (but we do not currently collect)", "captcha-selector": "div#captcha_container" } } From cfb261a989734b1a2bc648db155fe9103ba9bce0 Mon Sep 17 00:00:00 2001 From: Dale Wahl Date: Thu, 28 Nov 2024 12:10:28 +0100 Subject: [PATCH 05/54] douyin: capture ?modal_id= video links! overhauled embed extraction and allow multiple embedded objects to be parsed --- modules/douyin.js | 426 ++++++++++++++++++++++++++-------------------- 1 file changed, 238 insertions(+), 188 deletions(-) diff --git a/modules/douyin.js b/modules/douyin.js index 9850e7a..e082d51 100644 --- a/modules/douyin.js +++ b/modules/douyin.js @@ -7,7 +7,7 @@ zeeschuimer.register_module( return []; } - if(!response) { + if (!response) { return []; } @@ -17,248 +17,298 @@ zeeschuimer.register_module( */ let embedded_sigil_start = /(')[0].trim(); + let json_bit = line.split(prefix)[1].split('')[0].trim(); if (json_bit.endsWith(';')) { json_bit = json_bit.substring(0, -1); } - if (prefix.indexOf("additionalDataLoaded") !== -1) { - // remove trailing ) - json_bit = json_bit.slice(0, -1); - } else if (js_prefixes.length === 0) { + if (js_prefixes.length === 0) { // last prefix has some special handling // remove trailing stuff... json_bit = json_bit.split(']]}}')[0]; diff --git a/test.js b/test.js new file mode 100644 index 0000000..e69de29 From ac8be430d5fe0c6df67b372508778d4500a703ce Mon Sep 17 00:00:00 2001 From: Stijn Peeters Date: Thu, 20 Feb 2025 11:28:38 +0100 Subject: [PATCH 23/54] Fix #41 --- modules/linkedin.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/linkedin.js b/modules/linkedin.js index cb82689..d6a6416 100644 --- a/modules/linkedin.js +++ b/modules/linkedin.js @@ -37,7 +37,7 @@ zeeschuimer.register_module( } } - const eligible_list_types = ["feedDashMainFeedByMainFeed", "feedDashInterestUpdatesByInterestFeedByKeywords", "feedDashProfileUpdatesByMemberShareFeed", "searchDashClustersByAll"] + const eligible_list_types = ["feedDashMainFeedByMainFeed", "feedDashInterestUpdatesByInterestFeedByKeywords", "feedDashProfileUpdatesByMemberShareFeed", "searchDashClustersByAll", "feedDashUpdatesByPostSlug"] const uninteresting_list_types = ["*dashMySettings", "messagingDashMessagingSettings", "*searchDashSearchHome", "searchDashTypeaheadByGlobalTypeahead", "messagingDashAffiliatedMailboxesAll", "legoDashPageContentsByPageKeyAndSlotId", "searchDashFilterClustersByFilters"] for (const data_bit of data) { // now we have the data, try to parse it From 6fb25ecda40aa9fd1f27c0e26954b8a8a11c1695 Mon Sep 17 00:00:00 2001 From: Stijn Peeters Date: Thu, 20 Feb 2025 11:28:51 +0100 Subject: [PATCH 24/54] Fix LinkedIn capture from initial search result page --- modules/linkedin.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/linkedin.js b/modules/linkedin.js index d6a6416..61fc99e 100644 --- a/modules/linkedin.js +++ b/modules/linkedin.js @@ -60,9 +60,9 @@ zeeschuimer.register_module( item_index = data_bit["data"]["data"][k][elements_key]; location = `data.data.${k}.${elements_key}`; - if (typeof (item_index) !== 'string' && item_index && item_index[0]['items'] !== undefined) { + if (item_index && item_index[0]['items'] !== undefined) { // embedded results on search page - item_index = item_index[0]['items'].map(item => { + item_index = item_index[0]['items'].filter(i => i['item']['searchFeedUpdate']).map(item => { return item['item']['searchFeedUpdate']['*update']; }); } From 37828ceb2720c981cffc494434ceb0e8c5f72951 Mon Sep 17 00:00:00 2001 From: Stijn Peeters Date: Thu, 20 Feb 2025 11:29:15 +0100 Subject: [PATCH 25/54] Delete test.js --- test.js | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 test.js diff --git a/test.js b/test.js deleted file mode 100644 index e69de29..0000000 From 6bf4c72d79df5a174d4ed02451d02865187d7269 Mon Sep 17 00:00:00 2001 From: Stijn Peeters Date: Thu, 20 Feb 2025 11:45:40 +0100 Subject: [PATCH 26/54] Fix inclusion of embedded Pinterest items --- modules/pinterest.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/modules/pinterest.js b/modules/pinterest.js index 4bd3660..a418d97 100644 --- a/modules/pinterest.js +++ b/modules/pinterest.js @@ -62,7 +62,7 @@ zeeschuimer.register_module( } - return [...traverse_data(data, function (item, property) { + for(const pin of traverse_data(data, function (item, property) { if ( // post page recommendations: https://www.pinterest.com/pin/507921664242278249/ @@ -86,6 +86,10 @@ zeeschuimer.register_module( item['_zs-origin'] = 'json'; return item; } - })] + })) { + pins.push(pin); + } + + return pins; } ) \ No newline at end of file From 537ac67826ae666071b9a7cfa8e3abd5bd959bf5 Mon Sep 17 00:00:00 2001 From: Stijn Peeters Date: Thu, 20 Feb 2025 12:32:35 +0100 Subject: [PATCH 27/54] Allow ranges for expected amount of items in tests --- tests/test.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/tests/test.py b/tests/test.py index 71b8a7d..5a6a82c 100644 --- a/tests/test.py +++ b/tests/test.py @@ -186,6 +186,11 @@ except selenium_exceptions.NoSuchElementException: pass + # ranges are expressed as tuples/lists (inclusive) + # if it's not a range, make it one anyway + expected = [settings["expected"], settings["expected"]] if type(settings["expected"]) is not list else settings["expected"] + nice_expected = f"between {expected[0]:,} and {expected[1]:,}" if expected[0] != expected[1] else str(expected[0]) + # look in Zeeschuimer how many items have been captured safename = platform.replace(".", "").replace("-", "") driver.switch_to.window(handles[0]) @@ -211,13 +216,16 @@ msg = f"{indent} {str.rjust(str(num_items), 4, ' ')} items :: " if try_scrolling: msg += f" {str.rjust(str(num_after_scroll), 4, ' ')} after scroll :: " - if num_items >= settings["expected"] and num_after_scroll > num_items: + if num_items >= expected[0] and num_items <= expected[1] and num_after_scroll > num_items: msg += colored("[✓]", "green", attrs=["bold"]) + " as expected" passed += 1 - elif num_items < settings["expected"] and num_after_scroll > num_items: - msg += colored("[⋯]", "yellow", attrs=["bold"]) + f" expected {settings['expected']:,}, get fewer, but more after scrolling" + elif expected[0] > num_items and num_after_scroll > num_items: + msg += colored("[⋯]", "yellow", attrs=["bold"]) + f" expected {nice_expected}, get fewer, but more after scrolling" + warnings += 1 + elif num_items > expected[1] and num_after_scroll > num_items: + msg += colored("[⋯]", "yellow", attrs=["bold"]) + f" expected {nice_expected}, get more, but more after scrolling" warnings += 1 - elif num_items >= settings["expected"] and num_after_scroll == num_items: + elif num_items >= expected[0] and num_items <= expected[1] and num_after_scroll == num_items: msg += colored("[⋯]", "yellow", attrs=["bold"]) + f" as expected, but no increase after scrolling" warnings += 1 else: @@ -225,14 +233,14 @@ failed += 1 else: msg += f" no scrolling :: " - if num_items == settings["expected"]: + if num_items >= expected[0] and num_items <= expected[1]: msg += colored("[✓]", "green", attrs=["bold"]) + " as expected" passed += 1 - elif num_items > settings["expected"]: - msg += colored("[⋯]", "yellow", attrs=["bold"]) + f" expected {settings['expected']:,}, but got more" + elif num_items > expected[1]: + msg += colored("[⋯]", "yellow", attrs=["bold"]) + f" expected {nice_expected}, but got more" warnings += 1 else: - msg += colored("[⨯]", "red", attrs=["bold"]) + f" expected {settings['expected']:,}, but got fewer" + msg += colored("[⨯]", "red", attrs=["bold"]) + f" expected {nice_expected}, but got fewer" failed += 1 print(msg) From 11551be1e58fea151acc7c6959b78a8498d5725f Mon Sep 17 00:00:00 2001 From: Stijn Peeters Date: Thu, 20 Feb 2025 12:32:42 +0100 Subject: [PATCH 28/54] Update tests --- tests/tests.json | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/tests.json b/tests/tests.json index 101fe52..2613f12 100644 --- a/tests/tests.json +++ b/tests/tests.json @@ -2,7 +2,7 @@ "instagram.com": { "Front page": { "https://instagram.com": { - "expected": 4, + "expected": [2, 5], "more-after-scroll": false } }, @@ -32,13 +32,13 @@ }, "Tag overview page": { "https://www.instagram.com/explore/tags/blessed/": { - "expected": 28, - "more-after-scroll": false + "expected": 18, + "more-after-scroll": true } }, "Location overview page": { "https://www.instagram.com/explore/locations/290958191/hongerige-wolf-groningen-netherlands/": { - "expected": 91, + "expected": 12, "more-after-scroll": true, "wait": 10 } @@ -332,7 +332,7 @@ "pinterest.com": { "Explore (main page)": { "https://www.pinterest.com/ideas/": { - "expected": 25, + "expected": [20, 30], "wait": 10, "more-after-scroll": true } @@ -346,26 +346,26 @@ }, "Board": { "https://www.pinterest.com/nahidessa/spiritual-groups/": { - "expected": 15, + "expected": [15, 65], "more-after-scroll": true } }, "Post page": { "https://www.pinterest.com/pin/507921664242278249/": { - "expected": 25, + "expected": [20,35], "more-after-scroll": true, "comment": "Individual post page also loads a number of related posts immediately" } }, "Account page": { - "https://www.pinterest.com/walmart/": { - "expected": 24, + "https://nl.pinterest.com/Walmart/_created/": { + "expected": [20, 30], "more-after-scroll": true } }, "Search results": { "https://www.pinterest.com/search/pins/?q=Aesthetic%20vibes": { - "expected": 65, + "expected": [40, 45], "more-after-scroll": true } } From 4d1cedd5ef924f9e5c48f1019b59a12cc12f2f82 Mon Sep 17 00:00:00 2001 From: Stijn Peeters Date: Thu, 20 Feb 2025 12:33:11 +0100 Subject: [PATCH 29/54] Version +0.0.1 --- .zenodo.json | 2 +- manifest.json | 2 +- popup/interface.html | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.zenodo.json b/.zenodo.json index 5e1a997..7286f1f 100644 --- a/.zenodo.json +++ b/.zenodo.json @@ -3,7 +3,7 @@ "license": "MPL-2.0", "title": "Zeeschuimer", "upload_type": "software", - "version": "v1.12.1", + "version": "v1.12.2", "keywords": [ "scraping", "data capture", "4cat", "instagram", "tiktok" ], diff --git a/manifest.json b/manifest.json index c501e45..a6c47af 100644 --- a/manifest.json +++ b/manifest.json @@ -3,7 +3,7 @@ "description": "Collect data while browsing social media platforms and upload it for analysis later", "manifest_version": 2, "name": "Zeeschuimer", - "version": "1.12.1", + "version": "1.12.2", "homepage_url": "https://github.com/digitalmethodsinitiative/zeeschuimer", "browser_specific_settings": { diff --git a/popup/interface.html b/popup/interface.html index 0cc0e33..1220890 100644 --- a/popup/interface.html +++ b/popup/interface.html @@ -382,7 +382,7 @@

Zeeschuimer

- v1.12.1 + v1.12.2

Captured data objects

From 1e1e89500be4dc42e3a6dbee47791331873a3986 Mon Sep 17 00:00:00 2001 From: Stijn Peeters Date: Wed, 12 Mar 2025 11:10:13 +0100 Subject: [PATCH 30/54] RedNote comments module --- manifest.json | 3 +- modules/rednote-comments.js | 56 +++++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 1 deletion(-) create mode 100644 modules/rednote-comments.js diff --git a/manifest.json b/manifest.json index a6c47af..6ec9cad 100644 --- a/manifest.json +++ b/manifest.json @@ -47,7 +47,8 @@ "modules/truth.js", "modules/threads.js", "modules/pinterest.js", - "modules/rednote.js" + "modules/rednote.js", + "modules/rednote-comments.js" ] } } diff --git a/modules/rednote-comments.js b/modules/rednote-comments.js new file mode 100644 index 0000000..0aa0f0d --- /dev/null +++ b/modules/rednote-comments.js @@ -0,0 +1,56 @@ +zeeschuimer.register_module( + 'RedNote (comments)', + "xiaohongshu.com", + function (response, source_platform_url, source_url) { + let domain = source_platform_url.split("/")[2].toLowerCase().replace(/^www\./, ''); + if (!domain.endsWith('xiaohongshu.com')) { + return []; + } + + // empty request response? nothing to parse, return immediately + if(!response) { + return []; + } + + // we may have multiple objects to read from to look for rednote posts; collect them + let datas = []; + try { + // try to parse the request response as JSON; if it is JSON, add to the array + datas.push(JSON.parse(response)); + } catch (e) { + // if not, it's probably HTML, so look for JSON embedded in the HTML + if(response.indexOf('