diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..9134472 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,18 @@ +--- +name: Bug report +about: Something is not working or output seems wrong +title: '' +labels: '' +assignees: '' + +--- + +Before submitting: +- Is this an issue with the output of 4CAT? Then please make an issue here instead: https://github.com/digitalmethodsinitiative/4cat/issues +- Is this an issue with the output of Zeehaven? Then please make an issue here instead: https://github.com/PublicDataLab/zeehaven/issues + +*Describe your bug* +1. What version of Zeeschuimer are you using? You can find the version at the top of the interface, in the header. +2. What platform are your trying to capture data from? +3. Describe what should be collected, but is not (e.g. a specific post, or all posts from a specific user, or a particular type of item). +4. If possible, include a link to the page you are trying to capture data from. diff --git a/.zenodo.json b/.zenodo.json index a29aaf3..f8da883 100644 --- a/.zenodo.json +++ b/.zenodo.json @@ -3,7 +3,7 @@ "license": "MPL-2.0", "title": "Zeeschuimer", "upload_type": "software", - "version": "v1.11.1", + "version": "v1.13.4", "keywords": [ "scraping", "data capture", "4cat", "instagram", "tiktok" ], diff --git a/README.md b/README.md index bc94dd0..85ff94c 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,9 @@ Currently, it supports the following platforms: * [Imgur](https://imgur.com) * [Douyin](https://douyin.com) * [Gab](https://gab.com) +* [Truth Social](https://truth.social) +* [Pinterest](https://pinterest.com) +* [RedNote/Xiaohongshu](https://xiaohongshu.com) Platform support requires regular maintenance to keep up with changes to the platforms. If something does not work, we welcome issues and pull requests. See 'Limitations' below for some known limitations to data capture. @@ -40,19 +43,29 @@ in any Firefox-based browser. If you want to run the latest development version debugging console](https://www.youtube.com/watch?v=J7el77F1ckg) after cloning the repository locally. ## How to use -A [guide to using Zeeschuimer and 4CAT](https://tinyurl.com/nmrw-zeeschuimer-tiktok) is available. Basic instructions +A [guide to using Zeeschuimer and 4CAT](https://zeeschuimer.4cat.nl/) is available. Basic instructions are as follows: -Install the browser extension in a Firefox browser. A button with the Zeeschuimer logo (a 'Z') will appear in the -browser toolbar. Click it to open the Zeeschuimer interface. Enable capturing for the sites you want to capture from. +Install the browser extension in a Firefox browser. A button with the Zeeschuimer logo (Zeeschuimer's browser icon, a yellow 'Z' on a green background) will appear in the browser toolbar. Click it +to open the Zeeschuimer interface. Enable capturing for the sites you want to capture from. -Next, simply browse a supported platform's site. You will see the amount of items detected per platform increase as you -browse. When you have the items you need, you can export the data as an [ndjson](https://ndjson.org) file, or upload it -to a 4CAT instance where a 4CAT dataset will be created from the uploaded items. You can then run 4CAT's analytical +Note that after installation in Firefox, the extension icon may not be immediately visible in the toolbar. If you can't +find Zeeschuimer's icon, look for the 'Extensions' icon (a puzzle piece); clicking it will show all available extensions +that are not shown in the main browser toolbar. + +### Collecting data +Next, simply browse a supported platform's site. **Zeeschuimer collects data as your browser receives it**. It is +always best practices to refresh a page after you toggle collection on for a supported platform's site in the control +panel (any data loaded previously will not be collected until refreshed). You can force a refresh with Ctl + F5 on Windows +or Shift + Command + R on Mac. You will see the amount of items detected per platform increase as you browse. Toggle +collection off when done to avoid inadvertantly gathering undesired data. + +When you have the items you need, you can export the data as an [ndjson](https://ndjson.org) file, or upload +it to a 4CAT instance where a 4CAT dataset will be created from the uploaded items. You can then run 4CAT's analytical processors on the data. -To upload to 4CAT, copy the URL of the website of the 4CAT instance to the "4CAT instance" field at the top of -Zeeschuimer's interface. You can then use the "to 4CAT" button to create a new 4CAT dataset from the captured data. +**To upload to 4CAT, copy the URL of the website of the 4CAT instance to the "4CAT instance" field at the bottom of +Zeeschuimer's interface**. You can then use the "to 4CAT" button to create a new 4CAT dataset from the captured data. After uploading, Zeeschuimer will show you a link and the ten most recently uploaded datasets are shown at the bottom of the interface. @@ -77,6 +90,13 @@ platform. The following limitations are known: * *TikTok* items that cannot be captured: * Live streams +For some platforms, the level of detail of the data that can be collected depends on the page it is captured from: + +* *Pinterest* items may lack some metadata unless captured from the individual post's page, most notably the timestamp + of the post. +* *RedNote/Xiaohongshu* items will often lack the item's post description, timestamp, and video URL, unless captured by + opening the post's own page/clicking it in an overview. + Note that these are *known* limitations; data capture may break or change based on platform changes. Always cross-reference captured data with what you are seeing in your browser. diff --git a/images/platform-icons/pinterest.png b/images/platform-icons/pinterest.png new file mode 100644 index 0000000..35ebfbc Binary files /dev/null and b/images/platform-icons/pinterest.png differ diff --git a/images/platform-icons/xiaohongshu.png b/images/platform-icons/xiaohongshu.png new file mode 100644 index 0000000..779e285 Binary files /dev/null and b/images/platform-icons/xiaohongshu.png differ diff --git a/images/zeeschuimer-16.png b/images/zeeschuimer-16.png new file mode 100644 index 0000000..c82952a Binary files /dev/null and b/images/zeeschuimer-16.png differ diff --git a/js/lib.js b/js/lib.js new file mode 100644 index 0000000..cb81d49 --- /dev/null +++ b/js/lib.js @@ -0,0 +1,23 @@ +function traverse_data(obj, callback) { + let results = []; + + function _traverse_data(obj, callback) { + for (const property in obj) { + if (!obj.hasOwnProperty(property) || !obj[property]) { + // not actually a property + continue; + } + + let callback_result = callback(obj[property], property); + + if (callback_result) { + results.push(callback_result); + } else if (typeof (obj[property]) === "object") { + _traverse_data(obj[property], callback); + } + } + } + + _traverse_data(obj, callback); + return results; +} \ No newline at end of file diff --git a/js/zs-background.js b/js/zs-background.js index 54f6d9a..63c966b 100644 --- a/js/zs-background.js +++ b/js/zs-background.js @@ -84,7 +84,7 @@ window.zeeschuimer = { // the document can be parsed by all modules listening on either the origin or document's URL's domain let eligible_modules = Object.fromEntries(Object.entries(window.zeeschuimer.modules).filter(entry => { - return possible_source_domains.includes(entry[1]["domain"].toLowerCase()); + return possible_source_domains.some((domain) => domain.endsWith(entry[1]["domain"].toLowerCase())); })); filter.ondata = event => { @@ -95,15 +95,17 @@ window.zeeschuimer = { filter.onstop = async (event) => { // pass the document to all eligible modules that are also enabled + let enabled_modules = []; for(const module_id in eligible_modules) { const module_enabled_key = 'zs-enabled-' + module_id; let module_enabled = await browser.storage.local.get(module_enabled_key); module_enabled = module_enabled.hasOwnProperty(module_enabled_key) && !!parseInt(module_enabled[module_enabled_key]); if(module_enabled) { - await zeeschuimer.parse_request(full_response, origin_url, document_url, details.tabId); + enabled_modules.push(module_id); } } + await zeeschuimer.parse_request(full_response, origin_url, document_url, details.tabId, enabled_modules); filter.disconnect(); full_response = ''; } @@ -117,8 +119,9 @@ window.zeeschuimer = { * @param origin_url URL of the *page* the data was requested from * @param document_url URL of the content that was captured * @param tabId ID of the tab in which the request was captured + * @param enabled_modules List of IDs of enabled modules */ - parse_request: async function (response, origin_url, document_url, tabId) { + parse_request: async function (response, origin_url, document_url, tabId, enabled_modules) { if (!origin_url) { origin_url = document_url; } @@ -160,6 +163,10 @@ window.zeeschuimer = { let item_list = []; for (let module_id in this.modules) { + if(!enabled_modules.includes(module_id)) { + continue + } + item_list = this.modules[module_id].callback(response, origin_url, document_url); if (item_list && item_list.length > 0) { await Promise.all(item_list.map(async (item) => { diff --git a/manifest.json b/manifest.json index ffb9632..f6b20dc 100644 --- a/manifest.json +++ b/manifest.json @@ -3,12 +3,15 @@ "description": "Collect data while browsing social media platforms and upload it for analysis later", "manifest_version": 2, "name": "Zeeschuimer", - "version": "1.11.1", + "version": "1.13.4", "homepage_url": "https://github.com/digitalmethodsinitiative/zeeschuimer", "browser_specific_settings": { "gecko": { - "update_url": "https://extensions.digitalmethods.net/updates.json" + "update_url": "https://extensions.digitalmethods.net/updates.json", + "data_collection_permissions": { + "required": ["none"] + } } }, @@ -33,6 +36,7 @@ "scripts": [ "inc/dexie.js", "inc/he.js", + "js/lib.js", "js/zs-background.js", "modules/tiktok.js", "modules/tiktok-comments.js", @@ -44,7 +48,12 @@ "modules/douyin.js", "modules/gab.js", "modules/truth.js", - "modules/threads.js" + "modules/threads.js", + "modules/pinterest.js", + "modules/rednote.js", + "modules/rednote-comments.js", + "modules/facebook.js", + "modules/facebook-comments.js" ] } } diff --git a/modules/douyin.js b/modules/douyin.js index 7bcc2ac..aec6eed 100644 --- a/modules/douyin.js +++ b/modules/douyin.js @@ -7,7 +7,7 @@ zeeschuimer.register_module( return []; } - if(!response) { + if (!response) { return []; } @@ -17,225 +17,327 @@ zeeschuimer.register_module( */ let embedded_sigil_start = /(')[0].trim(); + let json_bit = line.split(prefix)[1].split('')[0].trim(); if (json_bit.endsWith(';')) { json_bit = json_bit.substring(0, -1); } - if (prefix.indexOf("additionalDataLoaded") !== -1) { - // remove trailing ) - json_bit = json_bit.slice(0, -1); - } else if (js_prefixes.length === 0) { + if (js_prefixes.length === 0) { // last prefix has some special handling // remove trailing stuff... json_bit = json_bit.split(']]}}')[0]; } + json_bit = json_bit.split('],["CometResourceScheduler"')[0]; + try { datas.push(JSON.parse(json_bit)); @@ -63,28 +61,13 @@ zeeschuimer.register_module( } } - const traverse = function (obj) { - for (const property in obj) { - if (!obj.hasOwnProperty(property)) { - // not actually a property - continue; - } + console.log('have ' + datas.length + ' datas from ' + source_platform_url) - if (property === 'post' && obj[property]['pk'] && obj[property]['code']) { - edges.push(obj[property]); - } else if (typeof (obj[property]) === "object") { - traverse(obj[property]); - } + return [...traverse_data(datas, function (item, property) { + if (property === 'post' && item['pk'] && item['code']) { + return item; } - } - - - for (const data of datas) { - if (data) { - traverse(data); - } - } - - return edges; - } + })] + }, + 'threads.net' ); \ No newline at end of file diff --git a/modules/tiktok.js b/modules/tiktok.js index 47ce125..ee0e1e1 100644 --- a/modules/tiktok.js +++ b/modules/tiktok.js @@ -24,6 +24,24 @@ zeeschuimer.register_module( from_embed = true; } + // NEW: detect JSON embedded in HTML via