{
 "schemaVersion": 1,
 "generated": "2026-10-30",
 "dataLicense": "CC BY-SA 4.0",
 "license": "CC BY-SA 4.0",
 "notice": "Data from Open Source Everything (https://opensourceeverything.org), licensed CC BY-SA 4.0: use it freely, credit the site, link to the page, and share changes under the same license. Listings were drafted with AI assistance and are managed by the r/TrueOpenSource moderators; check a project's own page before relying on a fact.",
 "id": "web-scraping",
 "name": "Web scraping",
 "page": "https://opensourceeverything.org/g/web-scraping/",
 "category": "dev-blocks",
 "categoryName": "Building blocks for apps",
 "job": "Collecting data from websites automatically",
 "closedApps": [
  "Apify",
  "Bright Data",
  "SerpApi",
  "ScrapingBee"
 ],
 "coverage": null,
 "alternatives": [
  {
   "id": "scrapy",
   "name": "Scrapy",
   "page": "https://opensourceeverything.org/p/scrapy/",
   "summary": "Scrapy is a free tool that automatically collects structured information from websites. It is for programmers and data people who want to build their own web data collection programs.",
   "kind": "developer_tool",
   "setup": "advanced",
   "setupWhy": "Made for developers: you use it from code or the command line.",
   "platforms": [
    "windows",
    "mac",
    "linux"
   ],
   "license": "BSD-3-Clause",
   "licenseClass": "open",
   "licenseNeedsCheck": false,
   "stars": 65076,
   "forks": null,
   "status": "active",
   "verified": false,
   "latestRelease": null,
   "code": "https://github.com/scrapy/scrapy",
   "website": "https://www.scrapy.org",
   "download": null,
   "release": null,
   "family": null,
   "familyChildren": [],
   "howToHelp": null,
   "project": "scrapy",
   "rank": 1,
   "fit": "full",
   "note": null,
   "why": "Does the whole job · Advanced setup",
   "score": 66.4
  },
  {
   "id": "crawl4ai",
   "name": "Crawl4AI",
   "page": "https://opensourceeverything.org/p/crawl4ai/",
   "summary": "Crawl4AI is a free tool that visits websites and copies their content into clean text that AI programs can read. It is for developers and researchers who want to collect web data automatically, and it also offers a paid hosted version so people can skip running their own browsers.",
   "kind": "developer_tool",
   "setup": "advanced",
   "setupWhy": "Made for developers: you use it from code or the command line.",
   "platforms": [
    "server"
   ],
   "license": "Apache-2.0",
   "licenseClass": "open",
   "licenseNeedsCheck": false,
   "stars": 86409,
   "forks": null,
   "status": "active",
   "verified": false,
   "latestRelease": null,
   "code": "https://github.com/unclecode/crawl4ai",
   "website": null,
   "download": null,
   "release": null,
   "family": null,
   "familyChildren": [],
   "howToHelp": null,
   "project": "crawl4ai",
   "rank": 2,
   "fit": "full",
   "note": null,
   "why": "Does the whole job · Advanced setup",
   "score": 64.8
  },
  {
   "id": "firecrawl",
   "name": "Firecrawl",
   "page": "https://opensourceeverything.org/p/firecrawl/",
   "summary": "Firecrawl is a service and open-source code that collects text and data from websites, including pages that rely on JavaScript, and formats it for AI tools and apps. It is aimed at developers and businesses building AI agents or data pipelines.",
   "kind": "self_hosted",
   "setup": "advanced",
   "setupWhy": "You run it on your own server (a computer you set up and keep running), or from the command line.",
   "platforms": [
    "server"
   ],
   "license": "AGPL-3.0",
   "licenseClass": "open",
   "licenseNeedsCheck": false,
   "stars": 196066,
   "forks": null,
   "status": "active",
   "verified": false,
   "latestRelease": null,
   "code": "https://github.com/firecrawl/firecrawl",
   "website": null,
   "download": null,
   "release": null,
   "family": null,
   "familyChildren": [],
   "howToHelp": null,
   "project": "firecrawl",
   "rank": 3,
   "fit": "full",
   "note": null,
   "why": "Does the whole job · Advanced setup",
   "score": 63
  },
  {
   "id": "colly",
   "name": "Colly",
   "page": "https://opensourceeverything.org/p/colly/",
   "summary": "Colly is a free toolkit for writing programs that automatically visit websites and pull out structured information from them. It is made for programmers who write code in the Go language.",
   "kind": "library",
   "setup": "advanced",
   "setupWhy": "A building block for developers, not a ready-to-use app.",
   "platforms": [],
   "license": "Apache-2.0",
   "licenseClass": "open",
   "licenseNeedsCheck": false,
   "stars": 25571,
   "forks": null,
   "status": "active",
   "verified": false,
   "latestRelease": null,
   "code": "https://github.com/gocolly/colly",
   "website": "https://go-colly.org",
   "download": null,
   "release": null,
   "family": null,
   "familyChildren": [],
   "howToHelp": null,
   "project": "colly",
   "rank": 4,
   "fit": "partial",
   "note": "A coding framework for programmers; no ready-made hosted service or point-and-click tool.",
   "why": "Does part of the job · Advanced setup",
   "score": 33.2
  }
 ],
 "source": "https://opensourceeverything.org/g/web-scraping/",
 "group": "Web scraping",
 "replaces": [
  "Apify",
  "Bright Data",
  "SerpApi",
  "ScrapingBee"
 ]
}